20:23:55.379 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:23:55.379 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task93Kond/test-tmp
20:23:55.379 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
20:23:55.379 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task93Kond/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
20:23:55.381 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:23:55.754 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:23:55.754 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:23:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:55.757323 10546 mini_oidc.cc:150] Starting JWKS server
20:23:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:55.759819 10546 webserver.cc:492] Webserver started at http://127.0.0.1:41397/ using document root <none> and password file <none>
20:23:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:55.760066 10546 mini_oidc.cc:186] Starting OIDC Discovery server
20:23:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:55.760645 10546 webserver.cc:492] Webserver started at http://127.0.0.1:38241/ using document root <none> and password file <none>
20:23:55.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T20:23:55Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:23:55.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T20:23:55Z Disabled control of system clock
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:55.800639 10546 external_mini_cluster.cc:1366] Running /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/wal
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/data
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/logs
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/data/info.pb
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.190:33413
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.190
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41397/jwks.json
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:23:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:23:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.161548 10557 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:23:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.161896 10557 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:23:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.161983 10557 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:23:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.170801 10557 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:23:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.170908 10557 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
20:23:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.170957 10557 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:23:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.170998 10557 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:23:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.171054 10557 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:23:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.171095 10557 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:23:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.171139 10557 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41397/jwks.json
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.181834 10557 master_runner.cc:386] Master server non-default flags:
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/data
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/wal
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.190:33413
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41397/jwks.json
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/data/info.pb
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.190
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/logs
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c56fb2464ecad91a58d4589e1075de20e0f75c71
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 20:03:19 UTC on 24a791456cd2
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8684
20:23:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:23:56.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.183594 10557 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:23:56.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.185618 10557 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:23:56.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.194851 10563 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:56.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.194893 10565 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:56.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.194934 10562 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:56.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.195813 10557 server_base.cc:1047] running on GCE node
20:23:56.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.196704 10557 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:23:56.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.199160 10557 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:23:56.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.200471 10557 hybrid_clock.cc:648] HybridClock initialized: now 1761251036200395 us; error 53 us; skew 500 ppm
20:23:56.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.200950 10557 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:23:56.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.203990 10557 webserver.cc:492] Webserver started at http://127.10.76.190:41085/ using document root <none> and password file <none>
20:23:56.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.204891 10557 fs_manager.cc:362] Metadata directory not provided
20:23:56.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.205034 10557 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:23:56.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.205421 10557 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:23:56.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.208048 10557 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/data/instance:
20:23:56.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d28077abe3e94ba1ab0fde10b0666fc1"
20:23:56.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:56 on dist-test-slave-s0pz"
20:23:56.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.209028 10557 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/wal/instance:
20:23:56.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d28077abe3e94ba1ab0fde10b0666fc1"
20:23:56.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:56 on dist-test-slave-s0pz"
20:23:56.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.215303 10557 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
20:23:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.219599 10571 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:23:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.221148 10557 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:23:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.221350 10557 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/data,/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/wal
20:23:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d28077abe3e94ba1ab0fde10b0666fc1"
20:23:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:56 on dist-test-slave-s0pz"
20:23:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.221616 10557 fs_report.cc:389] FS layout report
20:23:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:23:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/wal
20:23:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/wal
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/data/data
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:23:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:23:56.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.236415 10557 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:23:56.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.237468 10557 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:23:56.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.237810 10557 kserver.cc:163] Server-wide thread pool size limit: 3276
20:23:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.265372 10557 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.190:33413
20:23:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.265403 10632 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.76.190:33413 every 8 connection(s)
20:23:56.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.267056 10557 server_base.cc:1179] Dumped server information to /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/data/info.pb
20:23:56.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.271896 10633 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:23:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.276268 10546 external_mini_cluster.cc:1428] Started /tmp/dist-test-task93Kond/build/asan/bin/kudu as pid 10557
20:23:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.276659 10546 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-0/wal/instance
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.279906 10546 external_mini_cluster.cc:1366] Running /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/wal
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/data
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/logs
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/data/info.pb
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.189:33079
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.189
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41397/jwks.json
20:23:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:23:56.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:23:56.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.280113 10633 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } has no permanent_uuid. Determining permanent_uuid...
20:23:56.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.298254 10633 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } has no permanent_uuid. Determining permanent_uuid...
20:23:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.299696 10575 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.76.189:33079: connect: Connection refused (error 111)
20:23:56.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.301533 10633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.189:33079: Network error: Client connection negotiation failed: client connection to 127.10.76.189:33079: connect: Connection refused (error 111)
20:23:56.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.350966 10633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } attempt: 1
20:23:56.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.353124 10633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.189:33079: Network error: Client connection negotiation failed: client connection to 127.10.76.189:33079: connect: Connection refused (error 111)
20:23:56.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.421496 10633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } attempt: 2
20:23:56.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.424437 10633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.189:33079: Network error: Client connection negotiation failed: client connection to 127.10.76.189:33079: connect: Connection refused (error 111)
20:23:56.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.515879 10633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } attempt: 3
20:23:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.518432 10633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.189:33079: Network error: Client connection negotiation failed: client connection to 127.10.76.189:33079: connect: Connection refused (error 111)
20:23:56.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.650895 10637 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:23:56.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.651234 10637 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:23:56.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.651317 10637 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:23:56.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.659775 10637 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:23:56.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.659902 10637 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
20:23:56.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.659952 10637 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:23:56.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.659993 10637 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:23:56.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.660054 10637 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:23:56.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.660094 10637 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:23:56.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.660137 10637 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41397/jwks.json
20:23:56.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.661759 10633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } attempt: 4
20:23:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.663920 10633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.189:33079: Network error: Client connection negotiation failed: client connection to 127.10.76.189:33079: connect: Connection refused (error 111)
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.671623 10637 master_runner.cc:386] Master server non-default flags:
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/data
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/wal
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.189:33079
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:23:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41397/jwks.json
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/data/info.pb
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.189
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/logs
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c56fb2464ecad91a58d4589e1075de20e0f75c71
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 20:03:19 UTC on 24a791456cd2
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8684
20:23:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:23:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.673841 10637 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:23:56.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.675747 10637 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:23:56.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.685410 10643 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:56.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.685496 10646 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:56.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.685443 10644 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:56.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.690529 10637 server_base.cc:1047] running on GCE node
20:23:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.692006 10637 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:23:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.693750 10637 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:23:56.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.694998 10637 hybrid_clock.cc:648] HybridClock initialized: now 1761251036694929 us; error 48 us; skew 500 ppm
20:23:56.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.695329 10637 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:23:56.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.698293 10637 webserver.cc:492] Webserver started at http://127.10.76.189:46269/ using document root <none> and password file <none>
20:23:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.699093 10637 fs_manager.cc:362] Metadata directory not provided
20:23:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.699209 10637 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:23:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.699570 10637 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:23:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.702430 10637 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/data/instance:
20:23:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e33466ce148d4ff08b5bb972f2e2b1d1"
20:23:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:56 on dist-test-slave-s0pz"
20:23:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.703624 10637 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/wal/instance:
20:23:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e33466ce148d4ff08b5bb972f2e2b1d1"
20:23:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:56 on dist-test-slave-s0pz"
20:23:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.710467 10637 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.000s sys 0.008s
20:23:56.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.714565 10652 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:23:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.716240 10637 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
20:23:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.716452 10637 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/data,/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/wal
20:23:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e33466ce148d4ff08b5bb972f2e2b1d1"
20:23:56.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:56 on dist-test-slave-s0pz"
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.716717 10637 fs_report.cc:389] FS layout report
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/wal
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/wal
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/data/data
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:23:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:23:56.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.746223 10637 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:23:56.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.747256 10637 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:23:56.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.747596 10637 kserver.cc:163] Server-wide thread pool size limit: 3276
20:23:56.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.772881 10637 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.189:33079
20:23:56.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.772903 10713 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.76.189:33079 every 8 connection(s)
20:23:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.774551 10637 server_base.cc:1179] Dumped server information to /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/data/info.pb
20:23:56.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.779055 10714 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:23:56.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.783473 10546 external_mini_cluster.cc:1428] Started /tmp/dist-test-task93Kond/build/asan/bin/kudu as pid 10637
20:23:56.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.783685 10546 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-1/wal/instance
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.785359 10546 external_mini_cluster.cc:1366] Running /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/wal
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/data
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/logs
20:23:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/data/info.pb
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.188:37065
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.188
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41397/jwks.json
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
20:23:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.785480 10714 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } has no permanent_uuid. Determining permanent_uuid...
20:23:56.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.800701 10714 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } has no permanent_uuid. Determining permanent_uuid...
20:23:56.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.812633 10714 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } has no permanent_uuid. Determining permanent_uuid...
20:23:56.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.814009 10656 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:23:56.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.815915 10714 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.188:37065: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:23:56.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.865357 10714 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } attempt: 1
20:23:56.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.867436 10714 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.188:37065: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:23:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.935777 10714 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } attempt: 2
20:23:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.938711 10714 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.188:37065: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:23:56.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.963227 10633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } attempt: 5
20:23:56.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:56.975976 10633 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } has no permanent_uuid. Determining permanent_uuid...
20:23:56.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:56.978209 10633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.188:37065: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:23:57.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.029497 10633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } attempt: 1
20:23:57.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.030041 10714 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } attempt: 3
20:23:57.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.031749 10633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.188:37065: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:23:57.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.032068 10714 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.188:37065: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:23:57.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.099998 10633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } attempt: 2
20:23:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.102113 10633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.188:37065: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:23:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.155845 10717 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:23:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.156204 10717 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:23:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.156288 10717 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:23:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.164902 10717 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:23:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.165022 10717 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
20:23:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.165072 10717 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:23:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.165113 10717 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:23:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.165166 10717 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:23:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.165207 10717 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:23:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.165251 10717 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41397/jwks.json
20:23:57.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.175339 10714 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } attempt: 4
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.175905 10717 master_runner.cc:386] Master server non-default flags:
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/data
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/wal
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:23:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.188:37065
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41397/jwks.json
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/data/info.pb
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.188
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/logs
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c56fb2464ecad91a58d4589e1075de20e0f75c71
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 20:03:19 UTC on 24a791456cd2
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8684
20:23:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:23:57.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.178092 10717 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:23:57.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.178294 10714 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.188:37065: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:23:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.180035 10717 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:23:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.188838 10725 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.188791 10724 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.189749 10727 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:57.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.190410 10717 server_base.cc:1047] running on GCE node
20:23:57.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.191252 10717 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:23:57.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.192822 10717 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:23:57.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.194094 10717 hybrid_clock.cc:648] HybridClock initialized: now 1761251037194022 us; error 51 us; skew 500 ppm
20:23:57.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.194554 10717 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:23:57.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.197139 10717 webserver.cc:492] Webserver started at http://127.10.76.188:35761/ using document root <none> and password file <none>
20:23:57.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.197916 10717 fs_manager.cc:362] Metadata directory not provided
20:23:57.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.198026 10717 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:23:57.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.198374 10717 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:23:57.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.201045 10717 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/data/instance:
20:23:57.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79c4fa8a83cc427ba5ca48a8e88b1674"
20:23:57.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:57 on dist-test-slave-s0pz"
20:23:57.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.201893 10717 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/wal/instance:
20:23:57.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79c4fa8a83cc427ba5ca48a8e88b1674"
20:23:57.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:57 on dist-test-slave-s0pz"
20:23:57.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.207849 10717 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
20:23:57.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.208338 10633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } attempt: 3
20:23:57.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.210299 10633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.76.188:37065: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:23:57.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.212096 10733 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:23:57.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.213615 10717 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.213783 10717 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/data,/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/wal
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79c4fa8a83cc427ba5ca48a8e88b1674"
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:57 on dist-test-slave-s0pz"
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.213970 10717 fs_report.cc:389] FS layout report
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/wal
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/wal
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/data/data
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:23:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:23:57.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.228816 10717 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:23:57.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.229719 10717 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:23:57.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.230000 10717 kserver.cc:163] Server-wide thread pool size limit: 3276
20:23:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.255153 10717 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.188:37065
20:23:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.255177 10794 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.76.188:37065 every 8 connection(s)
20:23:57.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.256819 10717 server_base.cc:1179] Dumped server information to /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/data/info.pb
20:23:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.259734 10546 external_mini_cluster.cc:1428] Started /tmp/dist-test-task93Kond/build/asan/bin/kudu as pid 10717
20:23:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.259959 10546 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/master-2/wal/instance
20:23:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.262176 10795 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:23:57.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.271114 10795 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } has no permanent_uuid. Determining permanent_uuid...
20:23:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.293418 10795 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } has no permanent_uuid. Determining permanent_uuid...
20:23:57.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.300181 10795 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } has no permanent_uuid. Determining permanent_uuid...
20:23:57.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.311702 10795 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674: Bootstrap starting.
20:23:57.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.315256 10795 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674: Neither blocks nor log segments found. Creating new log.
20:23:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.316704 10795 log.cc:826] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674: Log is configured to *not* fsync() on all Append() calls
20:23:57.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.319979 10795 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674: No bootstrap required, opened a new log
20:23:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.328410 10795 raft_consensus.cc:359] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:23:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.328974 10795 raft_consensus.cc:385] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:23:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.329106 10795 raft_consensus.cc:740] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 79c4fa8a83cc427ba5ca48a8e88b1674, State: Initialized, Role: FOLLOWER
20:23:57.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.329973 10795 consensus_queue.cc:260] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:23:57.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.331094 10802 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:23:57.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.331413 10802 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [sys.catalog]: This master's current role is: FOLLOWER
20:23:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.332741 10795 sys_catalog.cc:565] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [sys.catalog]: configured and running, proceeding with master startup.
20:23:57.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.342446 10717 master_runner.cc:186] Error getting master registration for 127.10.76.190:33413: OK, instance_id { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" instance_seqno: 1761251036258638 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:23:57.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.350188 10717 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:23:57.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.351055 10813 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:23:57.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.351244 10813 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:23:57.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.387532 10633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } attempt: 4
20:23:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.399716 10633 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1: Bootstrap starting.
20:23:57.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.404465 10633 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1: Neither blocks nor log segments found. Creating new log.
20:23:57.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.406075 10633 log.cc:826] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1: Log is configured to *not* fsync() on all Append() calls
20:23:57.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.408174 10802 raft_consensus.cc:493] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:23:57.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.408444 10802 raft_consensus.cc:515] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:23:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.409641 10633 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1: No bootstrap required, opened a new log
20:23:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.410085 10802 leader_election.cc:290] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d28077abe3e94ba1ab0fde10b0666fc1 (127.10.76.190:33413), e33466ce148d4ff08b5bb972f2e2b1d1 (127.10.76.189:33079)
20:23:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.417991 10633 raft_consensus.cc:359] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:23:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.418409 10633 raft_consensus.cc:385] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:23:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.418538 10633 raft_consensus.cc:740] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d28077abe3e94ba1ab0fde10b0666fc1, State: Initialized, Role: FOLLOWER
20:23:57.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.419353 10633 consensus_queue.cc:260] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:23:57.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.420879 10816 sys_catalog.cc:455] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:23:57.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.421603 10816 sys_catalog.cc:458] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [sys.catalog]: This master's current role is: FOLLOWER
20:23:57.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.422160 10633 sys_catalog.cc:565] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [sys.catalog]: configured and running, proceeding with master startup.
20:23:57.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.422188 10598 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" is_pre_election: true
20:23:57.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.422855 10598 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 79c4fa8a83cc427ba5ca48a8e88b1674 in term 0.
20:23:57.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.423699 10734 leader_election.cc:304] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 79c4fa8a83cc427ba5ca48a8e88b1674, d28077abe3e94ba1ab0fde10b0666fc1; no voters:
20:23:57.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.424207 10802 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:23:57.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.424387 10802 raft_consensus.cc:493] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:23:57.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.424489 10802 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [term 0 FOLLOWER]: Advancing to term 1
20:23:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.427968 10802 raft_consensus.cc:515] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:23:57.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.429860 10802 leader_election.cc:290] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [CANDIDATE]: Term 1 election: Requested vote from peers d28077abe3e94ba1ab0fde10b0666fc1 (127.10.76.190:33413), e33466ce148d4ff08b5bb972f2e2b1d1 (127.10.76.189:33079)
20:23:57.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.431927 10598 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d28077abe3e94ba1ab0fde10b0666fc1"
20:23:57.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.432709 10598 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 0 FOLLOWER]: Advancing to term 1
20:23:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.438355 10598 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 79c4fa8a83cc427ba5ca48a8e88b1674 in term 1.
20:23:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.439071 10734 leader_election.cc:304] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 79c4fa8a83cc427ba5ca48a8e88b1674, d28077abe3e94ba1ab0fde10b0666fc1; no voters:
20:23:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.439478 10802 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [term 1 FOLLOWER]: Leader election won for term 1
20:23:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.440140 10802 raft_consensus.cc:697] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [term 1 LEADER]: Becoming Leader. State: Replica: 79c4fa8a83cc427ba5ca48a8e88b1674, State: Running, Role: LEADER
20:23:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.440822 10802 consensus_queue.cc:237] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:23:57.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.447887 10557 master_runner.cc:186] Error getting master registration for 127.10.76.189:33079: OK, instance_id { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" instance_seqno: 1761251036767028 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:23:57.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.459111 10557 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:23:57.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.459142 10822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 79c4fa8a83cc427ba5ca48a8e88b1674. Latest consensus state: current_term: 1 leader_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:23:57.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.459491 10822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [sys.catalog]: This master's current role is: LEADER
20:23:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.463480 10828 catalog_manager.cc:1568] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:23:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.463654 10828 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:23:57.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.468837 10831 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
20:23:57.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.471438 10831 catalog_manager.cc:1494] Initializing Kudu cluster ID...
20:23:57.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.477530 10714 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } attempt: 5
20:23:57.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.482232 10739 tablet.cc:2378] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:23:57.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.489360 10598 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 1 FOLLOWER]: Refusing update from remote peer 79c4fa8a83cc427ba5ca48a8e88b1674: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:23:57.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.490804 10822 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:23:57.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.494424 10714 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1: Bootstrap starting.
20:23:57.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:57.495045 10577 tablet.cc:2378] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:23:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.506784 10714 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1: Neither blocks nor log segments found. Creating new log.
20:23:57.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.509637 10714 log.cc:826] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1: Log is configured to *not* fsync() on all Append() calls
20:23:57.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.521066 10816 sys_catalog.cc:455] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 79c4fa8a83cc427ba5ca48a8e88b1674. Latest consensus state: current_term: 1 leader_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:23:57.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.521405 10816 sys_catalog.cc:458] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [sys.catalog]: This master's current role is: FOLLOWER
20:23:57.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.532030 10802 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:23:57.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.532393 10802 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [sys.catalog]: This master's current role is: LEADER
20:23:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.541299 10831 catalog_manager.cc:1357] Generated new cluster ID: df59c1a9fc8f4eaaa7dd1e9725c256e0
20:23:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.541433 10831 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
20:23:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.546254 10714 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1: No bootstrap required, opened a new log
20:23:57.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.553889 10816 sys_catalog.cc:455] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:23:57.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.554170 10816 sys_catalog.cc:458] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [sys.catalog]: This master's current role is: FOLLOWER
20:23:57.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.566098 10831 catalog_manager.cc:1380] Generated new certificate authority record
20:23:57.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.565837 10714 raft_consensus.cc:359] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:23:57.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.566620 10714 raft_consensus.cc:385] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:23:57.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.566891 10714 raft_consensus.cc:740] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e33466ce148d4ff08b5bb972f2e2b1d1, State: Initialized, Role: FOLLOWER
20:23:57.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.568375 10714 consensus_queue.cc:260] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:23:57.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.572283 10831 catalog_manager.cc:1514] Loading token signing keys...
20:23:57.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.585553 10838 sys_catalog.cc:455] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:23:57.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.586022 10838 sys_catalog.cc:458] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [sys.catalog]: This master's current role is: FOLLOWER
20:23:57.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.588989 10679 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" is_pre_election: true
20:23:57.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.589226 10678 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1"
20:23:57.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.591225 10677 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 0 FOLLOWER]: Advancing to term 1
20:23:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.598944 10677 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 1 FOLLOWER]: Refusing update from remote peer 79c4fa8a83cc427ba5ca48a8e88b1674: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:23:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.600106 10714 sys_catalog.cc:565] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [sys.catalog]: configured and running, proceeding with master startup.
20:23:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.600373 10802 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:23:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.605159 10831 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674: Generated new TSK 0
20:23:57.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.606555 10831 catalog_manager.cc:1524] Initializing in-progress tserver states...
20:23:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.629527 10838 sys_catalog.cc:455] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 79c4fa8a83cc427ba5ca48a8e88b1674. Latest consensus state: current_term: 1 leader_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:23:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.629833 10838 sys_catalog.cc:458] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [sys.catalog]: This master's current role is: FOLLOWER
20:23:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.630677 10822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:23:57.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.630965 10822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79c4fa8a83cc427ba5ca48a8e88b1674 [sys.catalog]: This master's current role is: LEADER
20:23:57.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.631445 10843 sys_catalog.cc:455] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:23:57.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.631724 10843 sys_catalog.cc:458] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [sys.catalog]: This master's current role is: FOLLOWER
20:23:57.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.636857 10546 external_mini_cluster.cc:1366] Running /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:57.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:57.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/wal
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/data
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/logs
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/data/info.pb
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.129:0
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.76.129
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.129
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:23:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
20:23:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.727929 10856 catalog_manager.cc:1269] Loaded cluster ID: df59c1a9fc8f4eaaa7dd1e9725c256e0
20:23:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.728114 10856 catalog_manager.cc:1562] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1: loading cluster ID for follower catalog manager: success
20:23:57.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.733052 10856 catalog_manager.cc:1584] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1: acquiring CA information for follower catalog manager: success
20:23:57.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:57.735816 10856 catalog_manager.cc:1612] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:23:58.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.208696 10850 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:23:58.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.209196 10850 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:23:58.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.209360 10850 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:23:58.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.218488 10850 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:23:58.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.218688 10850 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
20:23:58.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.218776 10850 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
20:23:58.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.218835 10850 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
20:23:58.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.218973 10850 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.76.129
20:23:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.230563 10850 tablet_server_runner.cc:78] Tablet server non-default flags:
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/data
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/wal
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.129:0
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/data/info.pb
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.129
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.76.129
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/logs
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:23:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:23:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c56fb2464ecad91a58d4589e1075de20e0f75c71
20:23:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:23:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 20:03:19 UTC on 24a791456cd2
20:23:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8684
20:23:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:23:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.232676 10850 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:23:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.234790 10850 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:23:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.245278 10863 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:58.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.245595 10862 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:58.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.248068 10865 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:58.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.248451 10850 server_base.cc:1047] running on GCE node
20:23:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.249347 10850 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:23:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.250608 10850 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:23:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.251868 10850 hybrid_clock.cc:648] HybridClock initialized: now 1761251038251760 us; error 91 us; skew 500 ppm
20:23:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.252224 10850 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:23:58.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.255033 10850 webserver.cc:492] Webserver started at http://127.10.76.129:46781/ using document root <none> and password file <none>
20:23:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.256045 10850 fs_manager.cc:362] Metadata directory not provided
20:23:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.256199 10850 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:23:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.256609 10850 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:23:58.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.259985 10850 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/data/instance:
20:23:58.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16644066851f423584704cdaf9b0098f"
20:23:58.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:58 on dist-test-slave-s0pz"
20:23:58.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.261163 10850 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/wal/instance:
20:23:58.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16644066851f423584704cdaf9b0098f"
20:23:58.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:58 on dist-test-slave-s0pz"
20:23:58.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.275072 10850 fs_manager.cc:696] Time spent creating directory manager: real 0.013s user 0.011s sys 0.001s
20:23:58.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.281179 10871 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:23:58.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.283413 10850 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
20:23:58.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.283649 10850 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/data,/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/wal
20:23:58.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16644066851f423584704cdaf9b0098f"
20:23:58.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:58 on dist-test-slave-s0pz"
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.283918 10850 fs_report.cc:389] FS layout report
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/wal
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/wal
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/data/data
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:23:58.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:23:58.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:23:58.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:23:58.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.325052 10850 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:23:58.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.326097 10850 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:23:58.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.326424 10850 kserver.cc:163] Server-wide thread pool size limit: 3276
20:23:58.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.330153 10850 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:23:58.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.330310 10850 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:23:58.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.330421 10850 ts_tablet_manager.cc:616] Registered 0 tablets
20:23:58.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.330495 10850 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:23:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.414209 10850 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.129:44833
20:23:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.414273 10995 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.76.129:44833 every 8 connection(s)
20:23:58.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.415773 10850 server_base.cc:1179] Dumped server information to /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/data/info.pb
20:23:58.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.418452 10546 external_mini_cluster.cc:1428] Started /tmp/dist-test-task93Kond/build/asan/bin/kudu as pid 10850
20:23:58.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.418697 10546 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-0/wal/instance
20:23:58.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.421104 10997 heartbeater.cc:344] Connected to a master server at 127.10.76.190:33413
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.423532 10546 external_mini_cluster.cc:1366] Running /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/wal
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/data
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/logs
20:23:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/data/info.pb
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.130:0
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.76.130
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.130
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:23:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
20:23:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.427696 10997 heartbeater.cc:461] Registering TS with master...
20:23:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.428565 10996 heartbeater.cc:344] Connected to a master server at 127.10.76.188:37065
20:23:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.428751 10996 heartbeater.cc:461] Registering TS with master...
20:23:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.429459 10998 heartbeater.cc:344] Connected to a master server at 127.10.76.189:33079
20:23:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.429515 10996 heartbeater.cc:507] Master 127.10.76.188:37065 requested a full tablet report, sending...
20:23:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.429653 10998 heartbeater.cc:461] Registering TS with master...
20:23:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.430181 10998 heartbeater.cc:507] Master 127.10.76.189:33079 requested a full tablet report, sending...
20:23:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.429459 10997 heartbeater.cc:507] Master 127.10.76.190:33413 requested a full tablet report, sending...
20:23:58.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.432067 10750 ts_manager.cc:194] Registered new tserver with Master: 16644066851f423584704cdaf9b0098f (127.10.76.129:44833)
20:23:58.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.432117 10669 ts_manager.cc:194] Registered new tserver with Master: 16644066851f423584704cdaf9b0098f (127.10.76.129:44833)
20:23:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.434165 10588 ts_manager.cc:194] Registered new tserver with Master: 16644066851f423584704cdaf9b0098f (127.10.76.129:44833)
20:23:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.434203 10750 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.76.129:60257
20:23:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.467137 10828 catalog_manager.cc:1269] Loaded cluster ID: df59c1a9fc8f4eaaa7dd1e9725c256e0
20:23:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.467274 10828 catalog_manager.cc:1562] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1: loading cluster ID for follower catalog manager: success
20:23:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.471261 10828 catalog_manager.cc:1584] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1: acquiring CA information for follower catalog manager: success
20:23:58.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.474045 10828 catalog_manager.cc:1612] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:23:58.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.796330 11001 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:23:58.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.796691 11001 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:23:58.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.796871 11001 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:23:58.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.806209 11001 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:23:58.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.806372 11001 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
20:23:58.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.806427 11001 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
20:23:58.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.806464 11001 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
20:23:58.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.806551 11001 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.76.130
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.816979 11001 tablet_server_runner.cc:78] Tablet server non-default flags:
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/data
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/wal
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.130:0
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/data/info.pb
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.130
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.76.130
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/logs
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c56fb2464ecad91a58d4589e1075de20e0f75c71
20:23:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:23:58.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 20:03:19 UTC on 24a791456cd2
20:23:58.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8684
20:23:58.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:23:58.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.818701 11001 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:23:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.820662 11001 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:23:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.831988 11007 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.832012 11006 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:58.832798 11009 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:58.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.833881 11001 server_base.cc:1047] running on GCE node
20:23:58.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.834605 11001 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:23:58.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.835708 11001 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:23:58.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.836947 11001 hybrid_clock.cc:648] HybridClock initialized: now 1761251038836877 us; error 42 us; skew 500 ppm
20:23:58.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.837312 11001 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:23:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.839955 11001 webserver.cc:492] Webserver started at http://127.10.76.130:39843/ using document root <none> and password file <none>
20:23:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.840816 11001 fs_manager.cc:362] Metadata directory not provided
20:23:58.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.840979 11001 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:23:58.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.841321 11001 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:23:58.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.843937 11001 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/data/instance:
20:23:58.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3c7768e6ba454b858fa713a8b1b15a31"
20:23:58.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:58 on dist-test-slave-s0pz"
20:23:58.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.845026 11001 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/wal/instance:
20:23:58.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3c7768e6ba454b858fa713a8b1b15a31"
20:23:58.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:58 on dist-test-slave-s0pz"
20:23:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.851208 11001 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
20:23:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.855194 11015 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.856830 11001 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.857055 11001 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/data,/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/wal
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3c7768e6ba454b858fa713a8b1b15a31"
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:58 on dist-test-slave-s0pz"
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.857293 11001 fs_report.cc:389] FS layout report
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/wal
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/wal
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/data/data
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:23:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:23:58.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.879305 11001 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:23:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.880277 11001 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:23:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.880604 11001 kserver.cc:163] Server-wide thread pool size limit: 3276
20:23:58.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.883952 11001 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:23:58.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.884106 11001 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:23:58.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.884238 11001 ts_tablet_manager.cc:616] Registered 0 tablets
20:23:58.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.884310 11001 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:23:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.949995 11001 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.130:36825
20:23:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.950066 11135 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.76.130:36825 every 8 connection(s)
20:23:58.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.951579 11001 server_base.cc:1179] Dumped server information to /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/data/info.pb
20:23:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.954732 11137 heartbeater.cc:344] Connected to a master server at 127.10.76.190:33413
20:23:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.955554 11137 heartbeater.cc:461] Registering TS with master...
20:23:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.957433 11137 heartbeater.cc:507] Master 127.10.76.190:33413 requested a full tablet report, sending...
20:23:58.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.960253 10588 ts_manager.cc:194] Registered new tserver with Master: 3c7768e6ba454b858fa713a8b1b15a31 (127.10.76.130:36825)
20:23:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.961266 10546 external_mini_cluster.cc:1428] Started /tmp/dist-test-task93Kond/build/asan/bin/kudu as pid 11001
20:23:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.961464 10546 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-1/wal/instance
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.965858 10546 external_mini_cluster.cc:1366] Running /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task93Kond/build/asan/bin/kudu
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/wal
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/data
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/logs
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/data/info.pb
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.131:0
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.76.131
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.131
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:23:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:23:58.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
20:23:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.973989 11136 heartbeater.cc:344] Connected to a master server at 127.10.76.188:37065
20:23:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.974078 11138 heartbeater.cc:344] Connected to a master server at 127.10.76.189:33079
20:23:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.974228 11136 heartbeater.cc:461] Registering TS with master...
20:23:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.974325 11138 heartbeater.cc:461] Registering TS with master...
20:23:58.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.975219 11136 heartbeater.cc:507] Master 127.10.76.188:37065 requested a full tablet report, sending...
20:23:58.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.975286 11138 heartbeater.cc:507] Master 127.10.76.189:33079 requested a full tablet report, sending...
20:23:58.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.976686 10750 ts_manager.cc:194] Registered new tserver with Master: 3c7768e6ba454b858fa713a8b1b15a31 (127.10.76.130:36825)
20:23:58.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.976796 10669 ts_manager.cc:194] Registered new tserver with Master: 3c7768e6ba454b858fa713a8b1b15a31 (127.10.76.130:36825)
20:23:58.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:58.977938 10750 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.76.130:39315
20:23:59.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.339965 11141 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:23:59.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.340337 11141 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:23:59.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.340498 11141 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:23:59.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.349459 11141 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:23:59.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.349668 11141 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
20:23:59.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.349751 11141 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
20:23:59.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.349808 11141 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
20:23:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.349946 11141 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.76.131
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.361155 11141 tablet_server_runner.cc:78] Tablet server non-default flags:
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.148:45937
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/data
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/wal
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.131:0
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/data/info.pb
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.131
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
20:23:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.76.190:33413,127.10.76.189:33079,127.10.76.188:37065
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.76.131
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/logs
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c56fb2464ecad91a58d4589e1075de20e0f75c71
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 20:03:19 UTC on 24a791456cd2
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8684
20:23:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
20:23:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.363216 11141 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:23:59.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.365237 11141 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:23:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.375682 11149 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.375718 11148 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.377148 11151 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:23:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.377782 11141 server_base.cc:1047] running on GCE node
20:23:59.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.378463 11141 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:23:59.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.379518 11141 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:23:59.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.380795 11141 hybrid_clock.cc:648] HybridClock initialized: now 1761251039380688 us; error 50 us; skew 500 ppm
20:23:59.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.381137 11141 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:23:59.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.383580 11141 webserver.cc:492] Webserver started at http://127.10.76.131:36379/ using document root <none> and password file <none>
20:23:59.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.384377 11141 fs_manager.cc:362] Metadata directory not provided
20:23:59.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.384526 11141 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:23:59.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.384967 11141 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:23:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.387904 11141 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/data/instance:
20:23:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a097ce4c3d574f719d9f97bca5b510d5"
20:23:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:59 on dist-test-slave-s0pz"
20:23:59.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.388984 11141 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/wal/instance:
20:23:59.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a097ce4c3d574f719d9f97bca5b510d5"
20:23:59.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:59 on dist-test-slave-s0pz"
20:23:59.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.395072 11141 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.004s
20:23:59.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.399333 11157 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.401075 11141 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.401309 11141 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/data,/tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/wal
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a097ce4c3d574f719d9f97bca5b510d5"
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 20:23:59 on dist-test-slave-s0pz"
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.401523 11141 fs_report.cc:389] FS layout report
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/wal
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/wal
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/data/data
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:23:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:23:59.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.418551 11141 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:23:59.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.419456 11141 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:23:59.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.419772 11141 kserver.cc:163] Server-wide thread pool size limit: 3276
20:23:59.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.423377 11141 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:23:59.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.423519 11141 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:23:59.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.423641 11141 ts_tablet_manager.cc:616] Registered 0 tablets
20:23:59.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.423842 11141 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
20:23:59.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.436848 10996 heartbeater.cc:499] Master 127.10.76.188:37065 was elected leader, sending a full tablet report...
20:23:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.489092 11141 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.131:40057
20:23:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.489146 11279 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.76.131:40057 every 8 connection(s)
20:23:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.490525 11141 server_base.cc:1179] Dumped server information to /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/data/info.pb
20:23:59.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.493978 11281 heartbeater.cc:344] Connected to a master server at 127.10.76.190:33413
20:23:59.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.494417 11281 heartbeater.cc:461] Registering TS with master...
20:23:59.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.494472 10546 external_mini_cluster.cc:1428] Started /tmp/dist-test-task93Kond/build/asan/bin/kudu as pid 11141
20:23:59.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.494642 10546 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task93Kond/test-tmp/mini-kudu-cluster8901797617280292348/ts-2/wal/instance
20:23:59.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.495523 11281 heartbeater.cc:507] Master 127.10.76.190:33413 requested a full tablet report, sending...
20:23:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.497957 10588 ts_manager.cc:194] Registered new tserver with Master: a097ce4c3d574f719d9f97bca5b510d5 (127.10.76.131:40057)
20:23:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.501771 11280 heartbeater.cc:344] Connected to a master server at 127.10.76.188:37065
20:23:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.501960 11280 heartbeater.cc:461] Registering TS with master...
20:23:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.502626 11280 heartbeater.cc:507] Master 127.10.76.188:37065 requested a full tablet report, sending...
20:23:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.503854 10749 ts_manager.cc:194] Registered new tserver with Master: a097ce4c3d574f719d9f97bca5b510d5 (127.10.76.131:40057)
20:23:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.504427 11282 heartbeater.cc:344] Connected to a master server at 127.10.76.189:33079
20:23:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.504545 11282 heartbeater.cc:461] Registering TS with master...
20:23:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.505088 11282 heartbeater.cc:507] Master 127.10.76.189:33079 requested a full tablet report, sending...
20:23:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.505095 10749 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.76.131:46839
20:23:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.506301 10668 ts_manager.cc:194] Registered new tserver with Master: a097ce4c3d574f719d9f97bca5b510d5 (127.10.76.131:40057)
20:23:59.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.521066 10546 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
20:23:59.523 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:23:59.523 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:23:59.524 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.76.190"
port: 33413
}
bound_http_address {
host: "127.10.76.190"
port: 41085
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.76.189"
port: 33079
}
bound_http_address {
host: "127.10.76.189"
port: 46269
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.76.188"
port: 37065
}
bound_http_address {
host: "127.10.76.188"
port: 35761
}
}
}
20:23:59.524 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:23:59.525 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.76.129"
port: 44833
}
bound_http_address {
host: "127.10.76.129"
port: 46781
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.76.130"
port: 36825
}
bound_http_address {
host: "127.10.76.130"
port: 39843
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.76.131"
port: 40057
}
bound_http_address {
host: "127.10.76.131"
port: 36379
}
}
}
20:23:59.525 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:23:59.527 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:23:59.529 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.76.190:33413(127.10.76.190:33413)] connecting to peer
20:23:59.531 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.76.190:33413(127.10.76.190:33413)] Successfully connected to peer
20:23:59.533 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.76.190
20:23:59.536 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:23:59.537 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.76.189:33079(127.10.76.189:33079)] connecting to peer
20:23:59.539 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.10.76.188:37065(127.10.76.188:37065)] Successfully connected to peer
20:23:59.541 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.76.188
20:23:59.544 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.76.189:33079(127.10.76.189:33079)] Successfully connected to peer
20:23:59.546 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0b5c1aa9, L:/127.0.0.1:43862 - R:/127.10.76.190:33413] using SASL/PLAIN
20:23:59.546 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.10.76.189
20:23:59.552 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa5605075, L:/127.0.0.1:49424 - R:/127.10.76.188:37065] using SASL/PLAIN
20:23:59.555 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:23:59.555 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=27), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [9ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [26ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: OK, deferred=null) after lookup
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.557794 10749 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:49424:
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:23:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:23:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:23:59.561 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf6b65705, L:/127.0.0.1:57036 - R:/127.10.76.189:33079] using SASL/PLAIN
20:23:59.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.560145 10749 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:23:59.577 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 04bc0a87ba2749dc8d8915144b505e4c
20:23:59.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.626314 11068 tablet_service.cc:1505] Processing CreateTablet for tablet 811bde3229e846daae3119f0b9c0e9f6 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=04bc0a87ba2749dc8d8915144b505e4c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
20:23:59.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.627389 10922 tablet_service.cc:1505] Processing CreateTablet for tablet 15b02b615684432e8d930fcc5a5d18f4 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=04bc0a87ba2749dc8d8915144b505e4c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
20:23:59.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.628080 11068 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 811bde3229e846daae3119f0b9c0e9f6. 1 dirs total, 0 dirs full, 0 dirs failed
20:23:59.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.628168 10921 tablet_service.cc:1505] Processing CreateTablet for tablet 811bde3229e846daae3119f0b9c0e9f6 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=04bc0a87ba2749dc8d8915144b505e4c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
20:23:59.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.629057 11069 tablet_service.cc:1505] Processing CreateTablet for tablet 15b02b615684432e8d930fcc5a5d18f4 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=04bc0a87ba2749dc8d8915144b505e4c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
20:23:59.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.629505 10922 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 15b02b615684432e8d930fcc5a5d18f4. 1 dirs total, 0 dirs full, 0 dirs failed
20:23:59.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.629876 11069 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 15b02b615684432e8d930fcc5a5d18f4. 1 dirs total, 0 dirs full, 0 dirs failed
20:23:59.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.630185 10921 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 811bde3229e846daae3119f0b9c0e9f6. 1 dirs total, 0 dirs full, 0 dirs failed
20:23:59.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.636996 11215 tablet_service.cc:1505] Processing CreateTablet for tablet 15b02b615684432e8d930fcc5a5d18f4 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=04bc0a87ba2749dc8d8915144b505e4c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
20:23:59.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.638727 11215 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 15b02b615684432e8d930fcc5a5d18f4. 1 dirs total, 0 dirs full, 0 dirs failed
20:23:59.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.650755 11214 tablet_service.cc:1505] Processing CreateTablet for tablet 811bde3229e846daae3119f0b9c0e9f6 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=04bc0a87ba2749dc8d8915144b505e4c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
20:23:59.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.651357 11214 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 811bde3229e846daae3119f0b9c0e9f6. 1 dirs total, 0 dirs full, 0 dirs failed
20:23:59.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.662460 11305 tablet_bootstrap.cc:492] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f: Bootstrap starting.
20:23:59.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.667151 11305 tablet_bootstrap.cc:654] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f: Neither blocks nor log segments found. Creating new log.
20:23:59.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.669536 11305 log.cc:826] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f: Log is configured to *not* fsync() on all Append() calls
20:23:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.675190 11305 tablet_bootstrap.cc:492] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f: No bootstrap required, opened a new log
20:23:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.675534 11305 ts_tablet_manager.cc:1403] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f: Time spent bootstrapping tablet: real 0.013s user 0.006s sys 0.004s
20:23:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.681897 11306 tablet_bootstrap.cc:492] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31: Bootstrap starting.
20:23:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.682827 11305 raft_consensus.cc:359] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.683321 11305 raft_consensus.cc:385] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:23:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.683331 11308 tablet_bootstrap.cc:492] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5: Bootstrap starting.
20:23:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.683452 11305 raft_consensus.cc:740] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 16644066851f423584704cdaf9b0098f, State: Initialized, Role: FOLLOWER
20:23:59.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.684299 11305 consensus_queue.cc:260] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.686352 11305 ts_tablet_manager.cc:1434] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f: Time spent starting tablet: real 0.011s user 0.004s sys 0.008s
20:23:59.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.686523 11306 tablet_bootstrap.cc:654] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31: Neither blocks nor log segments found. Creating new log.
20:23:59.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.687072 11308 tablet_bootstrap.cc:654] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5: Neither blocks nor log segments found. Creating new log.
20:23:59.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.688350 11306 log.cc:826] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31: Log is configured to *not* fsync() on all Append() calls
20:23:59.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.690186 11308 log.cc:826] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5: Log is configured to *not* fsync() on all Append() calls
20:23:59.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.693537 11305 tablet_bootstrap.cc:492] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f: Bootstrap starting.
20:23:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.695534 11305 tablet_bootstrap.cc:654] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f: Neither blocks nor log segments found. Creating new log.
20:23:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.697350 11308 tablet_bootstrap.cc:492] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5: No bootstrap required, opened a new log
20:23:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.697440 11306 tablet_bootstrap.cc:492] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31: No bootstrap required, opened a new log
20:23:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.697682 11308 ts_tablet_manager.cc:1403] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5: Time spent bootstrapping tablet: real 0.015s user 0.008s sys 0.003s
20:23:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.697746 11306 ts_tablet_manager.cc:1403] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31: Time spent bootstrapping tablet: real 0.016s user 0.009s sys 0.004s
20:23:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.697825 11305 tablet_bootstrap.cc:492] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f: No bootstrap required, opened a new log
20:23:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.698004 11305 ts_tablet_manager.cc:1403] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.003s
20:23:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.699097 11305 raft_consensus.cc:359] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.699313 11305 raft_consensus.cc:385] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:23:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.699405 11305 raft_consensus.cc:740] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 16644066851f423584704cdaf9b0098f, State: Initialized, Role: FOLLOWER
20:23:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.699685 11305 consensus_queue.cc:260] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.700219 11305 ts_tablet_manager.cc:1434] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
20:23:59.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.705272 11308 raft_consensus.cc:359] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.705652 11308 raft_consensus.cc:385] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:23:59.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.705750 11308 raft_consensus.cc:740] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a097ce4c3d574f719d9f97bca5b510d5, State: Initialized, Role: FOLLOWER
20:23:59.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.706437 11308 consensus_queue.cc:260] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.707108 11306 raft_consensus.cc:359] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.707510 11306 raft_consensus.cc:385] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:23:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.707645 11306 raft_consensus.cc:740] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3c7768e6ba454b858fa713a8b1b15a31, State: Initialized, Role: FOLLOWER
20:23:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.707697 11280 heartbeater.cc:499] Master 127.10.76.188:37065 was elected leader, sending a full tablet report...
20:23:59.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.708312 11306 consensus_queue.cc:260] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.708415 11308 ts_tablet_manager.cc:1434] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5: Time spent starting tablet: real 0.011s user 0.010s sys 0.000s
20:23:59.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.710274 11136 heartbeater.cc:499] Master 127.10.76.188:37065 was elected leader, sending a full tablet report...
20:23:59.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.711071 11306 ts_tablet_manager.cc:1434] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31: Time spent starting tablet: real 0.013s user 0.007s sys 0.003s
20:23:59.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.716063 11306 tablet_bootstrap.cc:492] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31: Bootstrap starting.
20:23:59.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.717157 11308 tablet_bootstrap.cc:492] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5: Bootstrap starting.
20:23:59.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.718082 11306 tablet_bootstrap.cc:654] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31: Neither blocks nor log segments found. Creating new log.
20:23:59.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.718995 11308 tablet_bootstrap.cc:654] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5: Neither blocks nor log segments found. Creating new log.
20:23:59.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.720903 11306 tablet_bootstrap.cc:492] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31: No bootstrap required, opened a new log
20:23:59.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.721043 11306 ts_tablet_manager.cc:1403] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
20:23:59.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.721264 11308 tablet_bootstrap.cc:492] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5: No bootstrap required, opened a new log
20:23:59.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.721432 11308 ts_tablet_manager.cc:1403] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.001s
20:23:59.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.721704 11306 raft_consensus.cc:359] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.721864 11306 raft_consensus.cc:385] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:23:59.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.721925 11306 raft_consensus.cc:740] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3c7768e6ba454b858fa713a8b1b15a31, State: Initialized, Role: FOLLOWER
20:23:59.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.722133 11306 consensus_queue.cc:260] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.722322 11308 raft_consensus.cc:359] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.722519 11308 raft_consensus.cc:385] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:23:59.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.722631 11306 ts_tablet_manager.cc:1434] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
20:23:59.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.722648 11308 raft_consensus.cc:740] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a097ce4c3d574f719d9f97bca5b510d5, State: Initialized, Role: FOLLOWER
20:23:59.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.722916 11308 consensus_queue.cc:260] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.724251 11308 ts_tablet_manager.cc:1434] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
20:23:59.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.729027 11139 tablet.cc:2378] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:23:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.743489 11283 tablet.cc:2378] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:23:59.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.746563 11309 raft_consensus.cc:493] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:23:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.746843 11309 raft_consensus.cc:515] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.748654 11309 leader_election.cc:290] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3c7768e6ba454b858fa713a8b1b15a31 (127.10.76.130:36825), a097ce4c3d574f719d9f97bca5b510d5 (127.10.76.131:40057)
20:23:59.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.757411 11091 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "811bde3229e846daae3119f0b9c0e9f6" candidate_uuid: "16644066851f423584704cdaf9b0098f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3c7768e6ba454b858fa713a8b1b15a31" is_pre_election: true
20:23:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.757525 11235 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "811bde3229e846daae3119f0b9c0e9f6" candidate_uuid: "16644066851f423584704cdaf9b0098f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a097ce4c3d574f719d9f97bca5b510d5" is_pre_election: true
20:23:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.758066 11235 raft_consensus.cc:2468] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 16644066851f423584704cdaf9b0098f in term 0.
20:23:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.758064 11091 raft_consensus.cc:2468] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 16644066851f423584704cdaf9b0098f in term 0.
20:23:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.758812 10874 leader_election.cc:304] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16644066851f423584704cdaf9b0098f, 3c7768e6ba454b858fa713a8b1b15a31; no voters:
20:23:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.759207 11309 raft_consensus.cc:2804] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Leader pre-election won for term 1
20:23:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.759374 11309 raft_consensus.cc:493] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:23:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.759487 11309 raft_consensus.cc:3060] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Advancing to term 1
20:23:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.761618 11309 raft_consensus.cc:515] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.762308 11309 leader_election.cc:290] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [CANDIDATE]: Term 1 election: Requested vote from peers 3c7768e6ba454b858fa713a8b1b15a31 (127.10.76.130:36825), a097ce4c3d574f719d9f97bca5b510d5 (127.10.76.131:40057)
20:23:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.763191 11235 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "811bde3229e846daae3119f0b9c0e9f6" candidate_uuid: "16644066851f423584704cdaf9b0098f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a097ce4c3d574f719d9f97bca5b510d5"
20:23:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.763307 11091 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "811bde3229e846daae3119f0b9c0e9f6" candidate_uuid: "16644066851f423584704cdaf9b0098f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3c7768e6ba454b858fa713a8b1b15a31"
20:23:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.763393 11235 raft_consensus.cc:3060] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Advancing to term 1
20:23:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.763569 11091 raft_consensus.cc:3060] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Advancing to term 1
20:23:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.765594 11235 raft_consensus.cc:2468] T 811bde3229e846daae3119f0b9c0e9f6 P a097ce4c3d574f719d9f97bca5b510d5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 16644066851f423584704cdaf9b0098f in term 1.
20:23:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.766099 11091 raft_consensus.cc:2468] T 811bde3229e846daae3119f0b9c0e9f6 P 3c7768e6ba454b858fa713a8b1b15a31 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 16644066851f423584704cdaf9b0098f in term 1.
20:23:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.766206 10874 leader_election.cc:304] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16644066851f423584704cdaf9b0098f, a097ce4c3d574f719d9f97bca5b510d5; no voters:
20:23:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.766588 11309 raft_consensus.cc:2804] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [term 1 FOLLOWER]: Leader election won for term 1
20:23:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.767079 11309 raft_consensus.cc:697] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [term 1 LEADER]: Becoming Leader. State: Replica: 16644066851f423584704cdaf9b0098f, State: Running, Role: LEADER
20:23:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.767633 11309 consensus_queue.cc:237] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:23:59.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:23:59.773051 10750 catalog_manager.cc:5649] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f reported cstate change: term changed from 0 to 1, leader changed from <none> to 16644066851f423584704cdaf9b0098f (127.10.76.129). New cstate: current_term: 1 leader_uuid: "16644066851f423584704cdaf9b0098f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } health_report { overall_health: UNKNOWN } } }
20:23:59.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:23:59.925226 10999 tablet.cc:2378] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:24:00.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.026437 11314 raft_consensus.cc:493] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:24:00.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.026713 11314 raft_consensus.cc:515] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.028439 11314 leader_election.cc:290] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 16644066851f423584704cdaf9b0098f (127.10.76.129:44833), a097ce4c3d574f719d9f97bca5b510d5 (127.10.76.131:40057)
20:24:00.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.033903 11313 raft_consensus.cc:493] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:24:00.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.034166 11313 raft_consensus.cc:515] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.036319 11313 leader_election.cc:290] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 16644066851f423584704cdaf9b0098f (127.10.76.129:44833), 3c7768e6ba454b858fa713a8b1b15a31 (127.10.76.130:36825)
20:24:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.038221 10951 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "15b02b615684432e8d930fcc5a5d18f4" candidate_uuid: "3c7768e6ba454b858fa713a8b1b15a31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "16644066851f423584704cdaf9b0098f" is_pre_election: true
20:24:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.038702 10951 raft_consensus.cc:2468] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3c7768e6ba454b858fa713a8b1b15a31 in term 0.
20:24:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.039038 11235 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "15b02b615684432e8d930fcc5a5d18f4" candidate_uuid: "3c7768e6ba454b858fa713a8b1b15a31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a097ce4c3d574f719d9f97bca5b510d5" is_pre_election: true
20:24:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.039425 11019 leader_election.cc:304] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16644066851f423584704cdaf9b0098f, 3c7768e6ba454b858fa713a8b1b15a31; no voters:
20:24:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.039507 11235 raft_consensus.cc:2468] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3c7768e6ba454b858fa713a8b1b15a31 in term 0.
20:24:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.039961 11314 raft_consensus.cc:2804] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:24:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.040136 11314 raft_consensus.cc:493] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:24:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.040275 11314 raft_consensus.cc:3060] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Advancing to term 1
20:24:00.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.043548 11314 raft_consensus.cc:515] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.044562 11314 leader_election.cc:290] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [CANDIDATE]: Term 1 election: Requested vote from peers 16644066851f423584704cdaf9b0098f (127.10.76.129:44833), a097ce4c3d574f719d9f97bca5b510d5 (127.10.76.131:40057)
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.045115 11235 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "15b02b615684432e8d930fcc5a5d18f4" candidate_uuid: "3c7768e6ba454b858fa713a8b1b15a31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a097ce4c3d574f719d9f97bca5b510d5"
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.045097 10951 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "15b02b615684432e8d930fcc5a5d18f4" candidate_uuid: "3c7768e6ba454b858fa713a8b1b15a31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "16644066851f423584704cdaf9b0098f"
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.045336 10951 raft_consensus.cc:3060] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Advancing to term 1
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.045364 11235 raft_consensus.cc:3060] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Advancing to term 1
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.046711 11091 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "15b02b615684432e8d930fcc5a5d18f4" candidate_uuid: "a097ce4c3d574f719d9f97bca5b510d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3c7768e6ba454b858fa713a8b1b15a31" is_pre_election: true
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.047096 11091 raft_consensus.cc:2393] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a097ce4c3d574f719d9f97bca5b510d5 in current term 1: Already voted for candidate 3c7768e6ba454b858fa713a8b1b15a31 in this term.
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.048390 11235 raft_consensus.cc:2468] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3c7768e6ba454b858fa713a8b1b15a31 in term 1.
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.048502 10951 raft_consensus.cc:2468] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3c7768e6ba454b858fa713a8b1b15a31 in term 1.
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.048936 11019 leader_election.cc:304] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16644066851f423584704cdaf9b0098f, 3c7768e6ba454b858fa713a8b1b15a31; no voters:
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.048799 10945 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "15b02b615684432e8d930fcc5a5d18f4" candidate_uuid: "a097ce4c3d574f719d9f97bca5b510d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "16644066851f423584704cdaf9b0098f" is_pre_election: true
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.049306 10945 raft_consensus.cc:2393] T 15b02b615684432e8d930fcc5a5d18f4 P 16644066851f423584704cdaf9b0098f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a097ce4c3d574f719d9f97bca5b510d5 in current term 1: Already voted for candidate 3c7768e6ba454b858fa713a8b1b15a31 in this term.
20:24:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.049280 11314 raft_consensus.cc:2804] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 1 FOLLOWER]: Leader election won for term 1
20:24:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.049887 11314 raft_consensus.cc:697] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [term 1 LEADER]: Becoming Leader. State: Replica: 3c7768e6ba454b858fa713a8b1b15a31, State: Running, Role: LEADER
20:24:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.050112 11161 leader_election.cc:304] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a097ce4c3d574f719d9f97bca5b510d5; no voters: 16644066851f423584704cdaf9b0098f, 3c7768e6ba454b858fa713a8b1b15a31
20:24:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.050539 11314 consensus_queue.cc:237] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.050973 11313 raft_consensus.cc:2749] T 15b02b615684432e8d930fcc5a5d18f4 P a097ce4c3d574f719d9f97bca5b510d5 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:24:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.056591 10750 catalog_manager.cc:5649] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3c7768e6ba454b858fa713a8b1b15a31 (127.10.76.130). New cstate: current_term: 1 leader_uuid: "3c7768e6ba454b858fa713a8b1b15a31" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } health_report { overall_health: UNKNOWN } } }
20:24:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.171811 11309 consensus_queue.cc:1048] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:24:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.179944 11319 consensus_queue.cc:1048] T 811bde3229e846daae3119f0b9c0e9f6 P 16644066851f423584704cdaf9b0098f [LEADER]: Connected to new peer: Peer: permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.353029 10750 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:49436:
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:24:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
20:24:00.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:00.354053 10750 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:24:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.366760 10922 tablet_service.cc:1505] Processing CreateTablet for tablet 1d18db91bb874003af2d011fc8cea3c0 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=36db19dc34714e328f37b945b87d7ca3]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
20:24:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.367118 11214 tablet_service.cc:1505] Processing CreateTablet for tablet 1d18db91bb874003af2d011fc8cea3c0 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=36db19dc34714e328f37b945b87d7ca3]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
20:24:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.367904 10922 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d18db91bb874003af2d011fc8cea3c0. 1 dirs total, 0 dirs full, 0 dirs failed
20:24:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.367904 11214 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d18db91bb874003af2d011fc8cea3c0. 1 dirs total, 0 dirs full, 0 dirs failed
20:24:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.373189 11068 tablet_service.cc:1505] Processing CreateTablet for tablet 1d18db91bb874003af2d011fc8cea3c0 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=36db19dc34714e328f37b945b87d7ca3]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
20:24:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.374269 11068 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d18db91bb874003af2d011fc8cea3c0. 1 dirs total, 0 dirs full, 0 dirs failed
20:24:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.375571 11344 tablet_bootstrap.cc:492] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f: Bootstrap starting.
20:24:00.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.377580 11344 tablet_bootstrap.cc:654] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f: Neither blocks nor log segments found. Creating new log.
20:24:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.380489 11344 tablet_bootstrap.cc:492] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f: No bootstrap required, opened a new log
20:24:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.380657 11344 ts_tablet_manager.cc:1403] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
20:24:00.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.381500 11346 tablet_bootstrap.cc:492] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31: Bootstrap starting.
20:24:00.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.381500 11344 raft_consensus.cc:359] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.381686 11344 raft_consensus.cc:385] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:24:00.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.381781 11344 raft_consensus.cc:740] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 16644066851f423584704cdaf9b0098f, State: Initialized, Role: FOLLOWER
20:24:00.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.382021 11344 consensus_queue.cc:260] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.382464 11319 tablet_replica.cc:442] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } } }
20:24:00.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.382722 11319 tablet_replica.cc:445] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f: This TxnStatusTablet replica's current role is: FOLLOWER
20:24:00.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.383862 11346 tablet_bootstrap.cc:654] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31: Neither blocks nor log segments found. Creating new log.
20:24:00.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.387169 11344 ts_tablet_manager.cc:1434] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
20:24:00.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.387799 11346 tablet_bootstrap.cc:492] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31: No bootstrap required, opened a new log
20:24:00.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.388093 11346 ts_tablet_manager.cc:1403] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31: Time spent bootstrapping tablet: real 0.007s user 0.004s sys 0.000s
20:24:00.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.388264 11347 tablet_bootstrap.cc:492] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: Bootstrap starting.
20:24:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.389065 11346 raft_consensus.cc:359] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.390164 11346 raft_consensus.cc:385] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:24:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.390278 11346 raft_consensus.cc:740] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3c7768e6ba454b858fa713a8b1b15a31, State: Initialized, Role: FOLLOWER
20:24:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.390442 11347 tablet_bootstrap.cc:654] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: Neither blocks nor log segments found. Creating new log.
20:24:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.390553 11346 consensus_queue.cc:260] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.391070 11314 tablet_replica.cc:442] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } } }
20:24:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.391287 11346 ts_tablet_manager.cc:1434] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
20:24:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.391361 11314 tablet_replica.cc:445] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31: This TxnStatusTablet replica's current role is: FOLLOWER
20:24:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.397300 11347 tablet_bootstrap.cc:492] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: No bootstrap required, opened a new log
20:24:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.397450 11347 ts_tablet_manager.cc:1403] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.004s
20:24:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.398118 11347 raft_consensus.cc:359] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.398267 11347 raft_consensus.cc:385] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:24:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.398326 11347 raft_consensus.cc:740] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a097ce4c3d574f719d9f97bca5b510d5, State: Initialized, Role: FOLLOWER
20:24:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.398536 11347 consensus_queue.cc:260] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.398988 11313 tablet_replica.cc:442] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } } }
20:24:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.399225 11313 tablet_replica.cc:445] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: This TxnStatusTablet replica's current role is: FOLLOWER
20:24:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.400023 11347 ts_tablet_manager.cc:1434] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
20:24:00.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.436319 11314 consensus_queue.cc:1048] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [LEADER]: Connected to new peer: Peer: permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:24:00.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.440881 11313 raft_consensus.cc:493] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:24:00.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.441088 11313 raft_consensus.cc:515] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.441897 11313 leader_election.cc:290] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 16644066851f423584704cdaf9b0098f (127.10.76.129:44833), 3c7768e6ba454b858fa713a8b1b15a31 (127.10.76.130:36825)
20:24:00.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.442504 10951 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "1d18db91bb874003af2d011fc8cea3c0" candidate_uuid: "a097ce4c3d574f719d9f97bca5b510d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "16644066851f423584704cdaf9b0098f" is_pre_election: true
20:24:00.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.442787 10951 raft_consensus.cc:2468] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a097ce4c3d574f719d9f97bca5b510d5 in term 0.
20:24:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.443046 11091 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "1d18db91bb874003af2d011fc8cea3c0" candidate_uuid: "a097ce4c3d574f719d9f97bca5b510d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3c7768e6ba454b858fa713a8b1b15a31" is_pre_election: true
20:24:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.443249 11161 leader_election.cc:304] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16644066851f423584704cdaf9b0098f, a097ce4c3d574f719d9f97bca5b510d5; no voters:
20:24:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.443295 11091 raft_consensus.cc:2468] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a097ce4c3d574f719d9f97bca5b510d5 in term 0.
20:24:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.443596 11313 raft_consensus.cc:2804] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:24:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.443718 11313 raft_consensus.cc:493] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:24:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.443832 11313 raft_consensus.cc:3060] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [term 0 FOLLOWER]: Advancing to term 1
20:24:00.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.446681 11313 raft_consensus.cc:515] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.446777 11325 consensus_queue.cc:1048] T 15b02b615684432e8d930fcc5a5d18f4 P 3c7768e6ba454b858fa713a8b1b15a31 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:24:00.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.447808 10951 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "1d18db91bb874003af2d011fc8cea3c0" candidate_uuid: "a097ce4c3d574f719d9f97bca5b510d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "16644066851f423584704cdaf9b0098f"
20:24:00.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.448071 10951 raft_consensus.cc:3060] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f [term 0 FOLLOWER]: Advancing to term 1
20:24:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.451467 10951 raft_consensus.cc:2468] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a097ce4c3d574f719d9f97bca5b510d5 in term 1.
20:24:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.451983 11161 leader_election.cc:304] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16644066851f423584704cdaf9b0098f, a097ce4c3d574f719d9f97bca5b510d5; no voters:
20:24:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.452322 11351 raft_consensus.cc:2804] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [term 1 FOLLOWER]: Leader election won for term 1
20:24:00.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.452948 11351 raft_consensus.cc:697] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [term 1 LEADER]: Becoming Leader. State: Replica: a097ce4c3d574f719d9f97bca5b510d5, State: Running, Role: LEADER
20:24:00.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.453569 11351 consensus_queue.cc:237] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } }
20:24:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.453977 11313 leader_election.cc:290] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [CANDIDATE]: Term 1 election: Requested vote from peers 16644066851f423584704cdaf9b0098f (127.10.76.129:44833), 3c7768e6ba454b858fa713a8b1b15a31 (127.10.76.130:36825)
20:24:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.454496 11091 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "1d18db91bb874003af2d011fc8cea3c0" candidate_uuid: "a097ce4c3d574f719d9f97bca5b510d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3c7768e6ba454b858fa713a8b1b15a31"
20:24:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.454747 11091 raft_consensus.cc:3060] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31 [term 0 FOLLOWER]: Advancing to term 1
20:24:00.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.457784 11091 raft_consensus.cc:2468] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a097ce4c3d574f719d9f97bca5b510d5 in term 1.
20:24:00.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.458933 11353 tablet_replica.cc:442] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: TxnStatusTablet state changed. Reason: New leader a097ce4c3d574f719d9f97bca5b510d5. Latest consensus state: current_term: 1 leader_uuid: "a097ce4c3d574f719d9f97bca5b510d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } } }
20:24:00.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.459275 11353 tablet_replica.cc:445] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: This TxnStatusTablet replica's current role is: LEADER
20:24:00.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.459548 10749 catalog_manager.cc:5649] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 reported cstate change: term changed from 0 to 1, leader changed from <none> to a097ce4c3d574f719d9f97bca5b510d5 (127.10.76.131). New cstate: current_term: 1 leader_uuid: "a097ce4c3d574f719d9f97bca5b510d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } health_report { overall_health: HEALTHY } } }
20:24:00.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.466583 11355 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
20:24:00.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.466915 11355 txn_status_manager.cc:930] Loading transaction status metadata into memory...
20:24:00.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.495239 11091 raft_consensus.cc:1275] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31 [term 1 FOLLOWER]: Refusing update from remote peer a097ce4c3d574f719d9f97bca5b510d5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:24:00.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.495260 10951 raft_consensus.cc:1275] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f [term 1 FOLLOWER]: Refusing update from remote peer a097ce4c3d574f719d9f97bca5b510d5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:24:00.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.496011 11353 consensus_queue.cc:1048] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:24:00.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.496412 11351 consensus_queue.cc:1048] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:24:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.514906 11325 tablet_replica.cc:442] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31: TxnStatusTablet state changed. Reason: New leader a097ce4c3d574f719d9f97bca5b510d5. Latest consensus state: current_term: 1 leader_uuid: "a097ce4c3d574f719d9f97bca5b510d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } } }
20:24:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.515220 11325 tablet_replica.cc:445] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31: This TxnStatusTablet replica's current role is: FOLLOWER
20:24:00.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.516609 11319 tablet_replica.cc:442] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f: TxnStatusTablet state changed. Reason: New leader a097ce4c3d574f719d9f97bca5b510d5. Latest consensus state: current_term: 1 leader_uuid: "a097ce4c3d574f719d9f97bca5b510d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } } }
20:24:00.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.516909 11319 tablet_replica.cc:445] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f: This TxnStatusTablet replica's current role is: FOLLOWER
20:24:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.522033 11332 mvcc.cc:204] Tried to move back new op lower bound from 7214084261858983936 to 7214084261703090176. Current Snapshot: MvccSnapshot[applied={T|T < 7214084261858983936}]
20:24:00.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.523288 11351 tablet_replica.cc:442] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a097ce4c3d574f719d9f97bca5b510d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } } }
20:24:00.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.523545 11351 tablet_replica.cc:445] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: This TxnStatusTablet replica's current role is: LEADER
20:24:00.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.528045 11313 tablet_replica.cc:442] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a097ce4c3d574f719d9f97bca5b510d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } } }
20:24:00.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.528483 11313 tablet_replica.cc:445] T 1d18db91bb874003af2d011fc8cea3c0 P a097ce4c3d574f719d9f97bca5b510d5: This TxnStatusTablet replica's current role is: LEADER
20:24:00.531 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
20:24:00.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.537364 11325 tablet_replica.cc:442] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a097ce4c3d574f719d9f97bca5b510d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } } }
20:24:00.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.537650 11325 tablet_replica.cc:445] T 1d18db91bb874003af2d011fc8cea3c0 P 3c7768e6ba454b858fa713a8b1b15a31: This TxnStatusTablet replica's current role is: FOLLOWER
20:24:00.538 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 04bc0a87ba2749dc8d8915144b505e4c
20:24:00.539 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.76.188:37065
20:24:00.539 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:24:00.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.540536 10546 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task93Kond/build/asan/bin/kudu pid 10717
20:24:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.542075 10790 generic_service.cc:196] Checking for leaks (request via RPC)
20:24:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.542317 11319 tablet_replica.cc:442] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a097ce4c3d574f719d9f97bca5b510d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "16644066851f423584704cdaf9b0098f" member_type: VOTER last_known_addr { host: "127.10.76.129" port: 44833 } } peers { permanent_uuid: "3c7768e6ba454b858fa713a8b1b15a31" member_type: VOTER last_known_addr { host: "127.10.76.130" port: 36825 } } peers { permanent_uuid: "a097ce4c3d574f719d9f97bca5b510d5" member_type: VOTER last_known_addr { host: "127.10.76.131" port: 40057 } } }
20:24:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.542626 11319 tablet_replica.cc:445] T 1d18db91bb874003af2d011fc8cea3c0 P 16644066851f423584704cdaf9b0098f: This TxnStatusTablet replica's current role is: FOLLOWER
20:24:00.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.676944 10546 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task93Kond/build/asan/bin/kudu with pid 10717
20:24:00.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:00.714329 10996 heartbeater.cc:646] Failed to heartbeat to 127.10.76.188:37065 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.10.76.188:37065 (error 108)
20:24:00.715 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:24:00.715 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.10.76.188:37065(127.10.76.188:37065)] handling channelInactive
20:24:00.715 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state READY due to: connection closed
20:24:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.977981 11365 raft_consensus.cc:493] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 79c4fa8a83cc427ba5ca48a8e88b1674)
20:24:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.977988 11364 raft_consensus.cc:493] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 79c4fa8a83cc427ba5ca48a8e88b1674)
20:24:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.978196 11365 raft_consensus.cc:515] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.978190 11364 raft_consensus.cc:515] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:00.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.979403 11365 leader_election.cc:290] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d28077abe3e94ba1ab0fde10b0666fc1 (127.10.76.190:33413), 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065)
20:24:00.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.979431 11364 leader_election.cc:290] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e33466ce148d4ff08b5bb972f2e2b1d1 (127.10.76.189:33079), 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065)
20:24:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.980252 10677 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" is_pre_election: true
20:24:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.980540 10677 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d28077abe3e94ba1ab0fde10b0666fc1 in term 1.
20:24:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.980654 10598 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" is_pre_election: true
20:24:00.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.980911 10598 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e33466ce148d4ff08b5bb972f2e2b1d1 in term 1.
20:24:00.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:00.981606 10656 leader_election.cc:336] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.981968 10575 leader_election.cc:304] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d28077abe3e94ba1ab0fde10b0666fc1, e33466ce148d4ff08b5bb972f2e2b1d1; no voters:
20:24:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.981971 10653 leader_election.cc:304] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d28077abe3e94ba1ab0fde10b0666fc1, e33466ce148d4ff08b5bb972f2e2b1d1; no voters: 79c4fa8a83cc427ba5ca48a8e88b1674
20:24:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.982357 11364 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:24:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.982434 11365 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:24:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.982504 11364 raft_consensus.cc:493] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 79c4fa8a83cc427ba5ca48a8e88b1674)
20:24:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.982571 11365 raft_consensus.cc:493] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 79c4fa8a83cc427ba5ca48a8e88b1674)
20:24:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.982615 11364 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 1 FOLLOWER]: Advancing to term 2
20:24:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.982666 11365 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 1 FOLLOWER]: Advancing to term 2
20:24:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:00.982987 10575 leader_election.cc:336] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.984855 11365 raft_consensus.cc:515] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.984875 11364 raft_consensus.cc:515] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.985461 11365 leader_election.cc:290] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 2 election: Requested vote from peers d28077abe3e94ba1ab0fde10b0666fc1 (127.10.76.190:33413), 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065)
20:24:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.985533 11364 leader_election.cc:290] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 2 election: Requested vote from peers e33466ce148d4ff08b5bb972f2e2b1d1 (127.10.76.189:33079), 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065)
20:24:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.986302 10598 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "d28077abe3e94ba1ab0fde10b0666fc1"
20:24:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.986415 10677 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1"
20:24:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.986598 10598 raft_consensus.cc:2393] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate e33466ce148d4ff08b5bb972f2e2b1d1 in current term 2: Already voted for candidate d28077abe3e94ba1ab0fde10b0666fc1 in this term.
20:24:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.986686 10677 raft_consensus.cc:2393] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate d28077abe3e94ba1ab0fde10b0666fc1 in current term 2: Already voted for candidate e33466ce148d4ff08b5bb972f2e2b1d1 in this term.
20:24:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:00.987203 10656 leader_election.cc:336] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.987442 10653 leader_election.cc:304] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e33466ce148d4ff08b5bb972f2e2b1d1; no voters: 79c4fa8a83cc427ba5ca48a8e88b1674, d28077abe3e94ba1ab0fde10b0666fc1
20:24:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:00.987701 10575 leader_election.cc:336] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.987762 11365 raft_consensus.cc:2749] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
20:24:00.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.987871 10575 leader_election.cc:304] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d28077abe3e94ba1ab0fde10b0666fc1; no voters: 79c4fa8a83cc427ba5ca48a8e88b1674, e33466ce148d4ff08b5bb972f2e2b1d1
20:24:00.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:00.988135 11364 raft_consensus.cc:2749] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
20:24:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:01.045218 11136 heartbeater.cc:646] Failed to heartbeat to 127.10.76.188:37065 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:01.049 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.050 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.050 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.050 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.10.76.188:37065(127.10.76.188:37065) for tablet Kudu Master: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.069 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.071 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.089 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.090 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.091 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.091 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.109 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.110 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.110 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.111 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.111 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.129 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.130 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.130 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.130 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.169 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.170 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.171 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.229 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.230 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.230 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.269 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.270 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.270 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.271 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.272 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.10.76.188:37065, [1ms] received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [122ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [122ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [181ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [181ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [221ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [222ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, deferred=Deferred@1145566757(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1797215806@38e12cd8 -> (continuation of Deferred@661606925 after retry RPC after error@1118380790) -> (continuation of Deferred@958665304 after retry RPC after error@143814216) -> (continuation of Deferred@1437645421 after retry RPC after error@1585814719) -> (continuation of Deferred@1202381797 after retry RPC after error@1548821878) -> (continuation of Deferred@216178499 after retry RPC after error@2121304580) -> (continuation of Deferred@1859517412 after retry RPC after error@997857757), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/2034592728@23fa9176 -> (continuation of Deferred@661606925 after retry RPC after error@1118380790) -> (continuation of Deferred@958665304 after retry RPC after error@143814216) -> (continuation of Deferred@1437645421 after retry RPC after error@1585814719) -> (continuation of Deferred@1202381797 after retry RPC after error@1548821878) -> (continuation of Deferred@216178499 after retry RPC after error@2121304580) -> (continuation of Deferred@1859517412 after retry RPC after error@997857757)))
20:24:01.272 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.10.76.188:37065, [1ms] received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [122ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [122ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [181ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [181ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [221ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [222ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, deferred=Deferred@1145566757(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1797215806@38e12cd8 -> (continuation of Deferred@661606925 after retry RPC after error@1118380790) -> (continuation of Deferred@958665304 after retry RPC after error@143814216) -> (continuation of Deferred@1437645421 after retry RPC after error@1585814719) -> (continuation of Deferred@1202381797 after retry RPC after error@1548821878) -> (continuation of Deferred@216178499 after retry RPC after error@2121304580) -> (continuation of Deferred@1859517412 after retry RPC after error@997857757), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/2034592728@23fa9176 -> (continuation of Deferred@661606925 after retry RPC after error@1118380790) -> (continuation of Deferred@958665304 after retry RPC after error@143814216) -> (continuation of Deferred@1437645421 after retry RPC after error@1585814719) -> (continuation of Deferred@1202381797 after retry RPC after error@1548821878) -> (continuation of Deferred@216178499 after retry RPC after error@2121304580) -> (continuation of Deferred@1859517412 after retry RPC after error@997857757)))
20:24:01.289 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.290 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.290 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.290 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.291 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.309 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.311 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.311 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.329 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.331 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.331 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.349 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.350 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.350 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.369 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.370 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.370 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.370 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.371 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.373865 11364 raft_consensus.cc:493] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:24:01.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.374070 11365 raft_consensus.cc:493] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:24:01.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.374109 11364 raft_consensus.cc:515] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:01.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.374312 11365 raft_consensus.cc:515] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:01.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.374683 11364 leader_election.cc:290] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers e33466ce148d4ff08b5bb972f2e2b1d1 (127.10.76.189:33079), 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065)
20:24:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.375008 11365 leader_election.cc:290] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d28077abe3e94ba1ab0fde10b0666fc1 (127.10.76.190:33413), 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065)
20:24:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.375509 10677 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" is_pre_election: true
20:24:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.375576 10598 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" is_pre_election: true
20:24:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.375741 10677 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d28077abe3e94ba1ab0fde10b0666fc1 in term 2.
20:24:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.375771 10598 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e33466ce148d4ff08b5bb972f2e2b1d1 in term 2.
20:24:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:01.376009 10575 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111) [suppressed 10 similar messages]
20:24:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.376539 10575 leader_election.cc:304] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d28077abe3e94ba1ab0fde10b0666fc1, e33466ce148d4ff08b5bb972f2e2b1d1; no voters:
20:24:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.376613 10653 leader_election.cc:304] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d28077abe3e94ba1ab0fde10b0666fc1, e33466ce148d4ff08b5bb972f2e2b1d1; no voters:
20:24:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:01.376782 10656 leader_election.cc:336] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.376920 11365 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 2 FOLLOWER]: Leader pre-election won for term 3
20:24:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.377027 11365 raft_consensus.cc:493] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:24:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.377137 11365 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 2 FOLLOWER]: Advancing to term 3
20:24:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.377180 11364 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 2 FOLLOWER]: Leader pre-election won for term 3
20:24:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.377288 11364 raft_consensus.cc:493] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:24:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:01.377327 10575 leader_election.cc:336] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.377396 11364 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 2 FOLLOWER]: Advancing to term 3
20:24:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.380100 11365 raft_consensus.cc:515] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.380100 11364 raft_consensus.cc:515] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:01.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.380930 11364 leader_election.cc:290] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 3 election: Requested vote from peers e33466ce148d4ff08b5bb972f2e2b1d1 (127.10.76.189:33079), 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065)
20:24:01.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.381280 11365 leader_election.cc:290] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 3 election: Requested vote from peers d28077abe3e94ba1ab0fde10b0666fc1 (127.10.76.190:33413), 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065)
20:24:01.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.381485 10598 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "d28077abe3e94ba1ab0fde10b0666fc1"
20:24:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.381631 10677 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1"
20:24:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.381745 10598 raft_consensus.cc:2393] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 3 FOLLOWER]: Leader election vote request: Denying vote to candidate e33466ce148d4ff08b5bb972f2e2b1d1 in current term 3: Already voted for candidate d28077abe3e94ba1ab0fde10b0666fc1 in this term.
20:24:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.381860 10677 raft_consensus.cc:2393] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 3 FOLLOWER]: Leader election vote request: Denying vote to candidate d28077abe3e94ba1ab0fde10b0666fc1 in current term 3: Already voted for candidate e33466ce148d4ff08b5bb972f2e2b1d1 in this term.
20:24:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:01.382467 10656 leader_election.cc:336] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.382649 10653 leader_election.cc:304] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [CANDIDATE]: Term 3 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e33466ce148d4ff08b5bb972f2e2b1d1; no voters: 79c4fa8a83cc427ba5ca48a8e88b1674, d28077abe3e94ba1ab0fde10b0666fc1
20:24:01.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:01.382725 10575 leader_election.cc:336] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:01.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.382840 10575 leader_election.cc:304] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 3 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d28077abe3e94ba1ab0fde10b0666fc1; no voters: 79c4fa8a83cc427ba5ca48a8e88b1674, e33466ce148d4ff08b5bb972f2e2b1d1
20:24:01.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.382942 11365 raft_consensus.cc:2749] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 3 FOLLOWER]: Leader election lost for term 3. Reason: could not achieve majority
20:24:01.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.383096 11364 raft_consensus.cc:2749] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 3 FOLLOWER]: Leader election lost for term 3. Reason: could not achieve majority
20:24:01.389 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.390 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.390 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.393 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.429 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.430 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.430 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.430 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.449 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.450 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.451 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.451 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.451 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.452 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [1ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [1ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [141ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [142ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, deferred=Deferred@845349760(state=PENDING, result=null, callback=(continuation of Deferred@461420718 after retry RPC after error@531741164) -> (continuation of Deferred@1718130315 after retry RPC after error@1279129285) -> (continuation of Deferred@1390830116 after retry RPC after error@1074450253) -> (continuation of Deferred@1651819785 after retry RPC after error@1104976988) -> (continuation of Deferred@541729025 after retry RPC after error@917304034) -> (continuation of Deferred@1916024243 after retry RPC after error@1471117781) -> (continuation of Deferred@647759880 after retry RPC after error@2004272603), errback=(continuation of Deferred@461420718 after retry RPC after error@531741164) -> (continuation of Deferred@1718130315 after retry RPC after error@1279129285) -> (continuation of Deferred@1390830116 after retry RPC after error@1074450253) -> (continuation of Deferred@1651819785 after retry RPC after error@1104976988) -> (continuation of Deferred@541729025 after retry RPC after error@917304034) -> (continuation of Deferred@1916024243 after retry RPC after error@1471117781) -> (continuation of Deferred@647759880 after retry RPC after error@2004272603)))
20:24:01.453 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [1ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [1ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [99ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [103ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [141ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [142ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, deferred=Deferred@845349760(state=PENDING, result=null, callback=(continuation of Deferred@461420718 after retry RPC after error@531741164) -> (continuation of Deferred@1718130315 after retry RPC after error@1279129285) -> (continuation of Deferred@1390830116 after retry RPC after error@1074450253) -> (continuation of Deferred@1651819785 after retry RPC after error@1104976988) -> (continuation of Deferred@541729025 after retry RPC after error@917304034) -> (continuation of Deferred@1916024243 after retry RPC after error@1471117781) -> (continuation of Deferred@647759880 after retry RPC after error@2004272603), errback=(continuation of Deferred@461420718 after retry RPC after error@531741164) -> (continuation of Deferred@1718130315 after retry RPC after error@1279129285) -> (continuation of Deferred@1390830116 after retry RPC after error@1074450253) -> (continuation of Deferred@1651819785 after retry RPC after error@1104976988) -> (continuation of Deferred@541729025 after retry RPC after error@917304034) -> (continuation of Deferred@1916024243 after retry RPC after error@1471117781) -> (continuation of Deferred@647759880 after retry RPC after error@2004272603)))
20:24:01.469 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.470 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.471 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.489 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.490 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.509 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.510 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.510 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.510 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.529 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.530 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.531 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.531 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.531 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:01.540025 11280 heartbeater.cc:646] Failed to heartbeat to 127.10.76.188:37065 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:01.549 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.551 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.551 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.551 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.569 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.629 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.630 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.630 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.631 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.686079 11164 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.151s from last keepalive heartbeat (effective timeout is 1.000s)
20:24:01.729 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [22ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [101ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [102ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, deferred=Deferred@828444666(state=PENDING, result=null, callback=(continuation of Deferred@129173329 after retry RPC after error@27810822) -> (continuation of Deferred@868957216 after retry RPC after error@174910332) -> (continuation of Deferred@360377583 after retry RPC after error@1458690602) -> (continuation of Deferred@1009895144 after retry RPC after error@1366027545) -> (continuation of Deferred@322207311 after retry RPC after error@1271711369) -> (continuation of Deferred@2046316446 after retry RPC after error@2004182685) -> (continuation of Deferred@1199814714 after retry RPC after error@1784075981), errback=(continuation of Deferred@129173329 after retry RPC after error@27810822) -> (continuation of Deferred@868957216 after retry RPC after error@174910332) -> (continuation of Deferred@360377583 after retry RPC after error@1458690602) -> (continuation of Deferred@1009895144 after retry RPC after error@1366027545) -> (continuation of Deferred@322207311 after retry RPC after error@1271711369) -> (continuation of Deferred@2046316446 after retry RPC after error@2004182685) -> (continuation of Deferred@1199814714 after retry RPC after error@1784075981)))
20:24:01.731 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
20:24:01.749 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.751 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.751 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.769 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.770 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.770 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.770 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.786646 11364 raft_consensus.cc:493] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:24:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.786898 11364 raft_consensus.cc:515] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.787732 11364 leader_election.cc:290] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers e33466ce148d4ff08b5bb972f2e2b1d1 (127.10.76.189:33079), 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065)
20:24:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.788408 10677 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" candidate_term: 4 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" is_pre_election: true
20:24:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.788712 10677 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d28077abe3e94ba1ab0fde10b0666fc1 in term 3.
20:24:01.789 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.789403 10575 leader_election.cc:304] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d28077abe3e94ba1ab0fde10b0666fc1, e33466ce148d4ff08b5bb972f2e2b1d1; no voters:
20:24:01.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:01.789739 10575 leader_election.cc:336] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:01.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.789814 11364 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 3 FOLLOWER]: Leader pre-election won for term 4
20:24:01.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.789979 11364 raft_consensus.cc:493] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:24:01.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.790091 11364 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 3 FOLLOWER]: Advancing to term 4
20:24:01.790 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.790 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.791 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.792630 11364 raft_consensus.cc:515] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.793243 11364 leader_election.cc:290] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 4 election: Requested vote from peers e33466ce148d4ff08b5bb972f2e2b1d1 (127.10.76.189:33079), 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065)
20:24:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.793828 10677 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" candidate_term: 4 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1"
20:24:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.794061 10677 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 3 FOLLOWER]: Advancing to term 4
20:24:01.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:01.794973 10575 leader_election.cc:336] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111)
20:24:01.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.796363 10677 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d28077abe3e94ba1ab0fde10b0666fc1 in term 4.
20:24:01.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.796903 10575 leader_election.cc:304] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d28077abe3e94ba1ab0fde10b0666fc1, e33466ce148d4ff08b5bb972f2e2b1d1; no voters: 79c4fa8a83cc427ba5ca48a8e88b1674
20:24:01.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.797215 11364 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 4 FOLLOWER]: Leader election won for term 4
20:24:01.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.797610 11364 raft_consensus.cc:697] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [term 4 LEADER]: Becoming Leader. State: Replica: d28077abe3e94ba1ab0fde10b0666fc1, State: Running, Role: LEADER
20:24:01.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.798056 11364 consensus_queue.cc:237] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.799800 11387 sys_catalog.cc:455] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d28077abe3e94ba1ab0fde10b0666fc1. Latest consensus state: current_term: 4 leader_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:24:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.800149 11387 sys_catalog.cc:458] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [sys.catalog]: This master's current role is: LEADER
20:24:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.801268 11389 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
20:24:01.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.804443 11389 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=04bc0a87ba2749dc8d8915144b505e4c]
20:24:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.805315 11389 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=36db19dc34714e328f37b945b87d7ca3]
20:24:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.807544 11389 tablet_loader.cc:96] loaded metadata for tablet 15b02b615684432e8d930fcc5a5d18f4 (table txn_manager_fallback_rolling [id=04bc0a87ba2749dc8d8915144b505e4c])
20:24:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.807981 11389 tablet_loader.cc:96] loaded metadata for tablet 1d18db91bb874003af2d011fc8cea3c0 (table kudu_system.kudu_transactions [id=36db19dc34714e328f37b945b87d7ca3])
20:24:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.808297 11389 tablet_loader.cc:96] loaded metadata for tablet 811bde3229e846daae3119f0b9c0e9f6 (table txn_manager_fallback_rolling [id=04bc0a87ba2749dc8d8915144b505e4c])
20:24:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.808538 11389 catalog_manager.cc:1494] Initializing Kudu cluster ID...
20:24:01.809 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.809921 11389 catalog_manager.cc:1269] Loaded cluster ID: df59c1a9fc8f4eaaa7dd1e9725c256e0
20:24:01.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.810165 11389 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
20:24:01.810 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079; will retry
20:24:01.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.811971 11389 catalog_manager.cc:1514] Loading token signing keys...
20:24:01.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.812927 11389 catalog_manager.cc:6033] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1: Loaded TSK: 0
20:24:01.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.813781 11389 catalog_manager.cc:1524] Initializing in-progress tserver states...
20:24:01.828 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.10.76.188:37065(127.10.76.188:37065)] connecting to peer
20:24:01.830 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.830 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.188:37065(127.10.76.188:37065)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.830 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.188:37065: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065
20:24:01.832 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:24:01.832 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=5, TimeoutTracker(timeout=250, elapsed=83), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [41ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [62ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.10.76.190:33413,127.10.76.188:37065,127.10.76.189:33079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.190:33413, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.188:37065, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.189:33079, [81ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.188:37065: Network error: Failed to connect to peer master-127.10.76.188:37065(127.10.76.188:37065): Connection refused: /127.10.76.188:37065, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.189:33079: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.190:33413: OK, deferred=Deferred@273393591(state=PENDING, result=null, callback=(continuation of Deferred@1909619960 after retry RPC after error@1523514773) -> (continuation of Deferred@2005035856 after retry RPC after error@976817160) -> (continuation of Deferred@1492029963 after retry RPC after error@1977741014) -> (continuation of Deferred@291361745 after retry RPC after error@1668727998), errback=(continuation of Deferred@1909619960 after retry RPC after error@1523514773) -> (continuation of Deferred@2005035856 after retry RPC after error@976817160) -> (continuation of Deferred@1492029963 after retry RPC after error@1977741014) -> (continuation of Deferred@291361745 after retry RPC after error@1668727998))) after lookup
20:24:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.876971 10587 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:43882:
20:24:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
20:24:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:24:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
20:24:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
20:24:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
20:24:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:24:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
20:24:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:01.878980 10587 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:24:01.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:01.883836 10575 consensus_peers.cc:597] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 -> Peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Couldn't send request to peer 79c4fa8a83cc427ba5ca48a8e88b1674. Status: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:24:01.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.886845 10677 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 4 FOLLOWER]: Refusing update from remote peer d28077abe3e94ba1ab0fde10b0666fc1: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 4 index: 12. (index mismatch)
20:24:01.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.887745 11387 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
20:24:01.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.890005 11400 sys_catalog.cc:455] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d28077abe3e94ba1ab0fde10b0666fc1. Latest consensus state: current_term: 4 leader_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:24:01.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.890326 11400 sys_catalog.cc:458] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [sys.catalog]: This master's current role is: FOLLOWER
20:24:01.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.892082 11364 sys_catalog.cc:455] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:24:01.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.892431 11364 sys_catalog.cc:458] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 [sys.catalog]: This master's current role is: LEADER
20:24:01.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.892818 11400 sys_catalog.cc:455] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } } }
20:24:01.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:01.893066 11400 sys_catalog.cc:458] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [sys.catalog]: This master's current role is: FOLLOWER
20:24:01.902 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: a097ce4c3d574f719d9f97bca5b510d5 (127.10.76.131:40057): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1761251040 last_transition_timestamp: 1761251041
20:24:02.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:02.371162 10575 consensus_peers.cc:597] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 -> Peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Couldn't send request to peer 79c4fa8a83cc427ba5ca48a8e88b1674. Status: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:24:02.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:02.465340 10557 master_runner.cc:186] Error getting master registration for 127.10.76.188:37065: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111),
20:24:02.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:02.465572 10557 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:24:02.720 [DEBUG - main] (Connection.java:575) [peer master-127.10.76.190:33413(127.10.76.190:33413)] disconnecting while in state READY
20:24:02.721 [DEBUG - main] (Connection.java:575) [peer master-127.10.76.189:33079(127.10.76.189:33079)] disconnecting while in state READY
20:24:02.721 [DEBUG - main] (Connection.java:575) [peer master-127.10.76.188:37065(127.10.76.188:37065)] disconnecting while in state TERMINATED
20:24:02.722 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:24:02.722 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.76.190:33413(127.10.76.190:33413)] handling channelInactive
20:24:02.722 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.190:33413(127.10.76.190:33413)] cleaning up while in state READY due to: connection closed
20:24:02.723 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.76.189:33079(127.10.76.189:33079)] handling channelInactive
20:24:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:02.722882 10546 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task93Kond/build/asan/bin/kudu pid 10850
20:24:02.723 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.189:33079(127.10.76.189:33079)] cleaning up while in state READY due to: connection closed
20:24:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:02.731175 10991 generic_service.cc:196] Checking for leaks (request via RPC)
20:24:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 20:24:02.823745 10575 consensus_peers.cc:597] T 00000000000000000000000000000000 P d28077abe3e94ba1ab0fde10b0666fc1 -> Peer 79c4fa8a83cc427ba5ca48a8e88b1674 (127.10.76.188:37065): Couldn't send request to peer 79c4fa8a83cc427ba5ca48a8e88b1674. Status: Network error: Client connection negotiation failed: client connection to 127.10.76.188:37065: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
20:24:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:02.837922 10546 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task93Kond/build/asan/bin/kudu with pid 10850
20:24:02.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:02.861449 10546 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task93Kond/build/asan/bin/kudu pid 11001
20:24:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:02.869659 11131 generic_service.cc:196] Checking for leaks (request via RPC)
20:24:02.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:02.975045 10546 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task93Kond/build/asan/bin/kudu with pid 11001
20:24:02.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:02.996219 10546 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task93Kond/build/asan/bin/kudu pid 11141
20:24:03.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:03.003665 11275 generic_service.cc:196] Checking for leaks (request via RPC)
20:24:03.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:03.129142 10546 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task93Kond/build/asan/bin/kudu with pid 11141
20:24:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:03.152997 10546 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task93Kond/build/asan/bin/kudu pid 10557
20:24:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:03.154386 10628 generic_service.cc:196] Checking for leaks (request via RPC)
20:24:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:03.275106 10546 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task93Kond/build/asan/bin/kudu with pid 10557
20:24:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:03.299863 10546 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task93Kond/build/asan/bin/kudu pid 10637
20:24:03.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:03.301138 10709 generic_service.cc:196] Checking for leaks (request via RPC)
20:24:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:03.415357 11422 raft_consensus.cc:493] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d28077abe3e94ba1ab0fde10b0666fc1)
20:24:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:03.415629 11422 raft_consensus.cc:515] T 00000000000000000000000000000000 P e33466ce148d4ff08b5bb972f2e2b1d1 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d28077abe3e94ba1ab0fde10b0666fc1" member_type: VOTER last_known_addr { host: "127.10.76.190" port: 33413 } } peers { permanent_uuid: "e33466ce148d4ff08b5bb972f2e2b1d1" member_type: VOTER last_known_addr { host: "127.10.76.189" port: 33079 } } peers { permanent_uuid: "79c4fa8a83cc427ba5ca48a8e88b1674" member_type: VOTER last_known_addr { host: "127.10.76.188" port: 37065 } }
20:24:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 20:24:03.415958 10546 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task93Kond/build/asan/bin/kudu with pid 10637
20:24:03.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T20:24:03Z chronyd exiting
20:24:03.581 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)