14:03:34.577 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:03:34.577 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLQ_Kwd/test-tmp
14:03:34.577 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
14:03:34.577 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLQ_Kwd/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
14:03:34.578 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:03:34.891 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:03:34.891 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:03:34.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:34.893683 21997 mini_oidc.cc:150] Starting JWKS server
14:03:34.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:34.896385 21997 webserver.cc:469] Webserver started at http://127.0.0.1:41795/ using document root <none> and password file <none>
14:03:34.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:34.896631 21997 mini_oidc.cc:186] Starting OIDC Discovery server
14:03:34.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:34.897181 21997 webserver.cc:469] Webserver started at http://127.0.0.1:45879/ using document root <none> and password file <none>
14:03:34.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-23T14:03:34Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:03:34.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-23T14:03:34Z Disabled control of system clock
14:03:34.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:34.926584 21997 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:34.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:34.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal
14:03:34.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/logs
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data/info.pb
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.126:34049
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.126
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41795/jwks.json
14:03:34.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:03:34.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:03:35.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.233902 22008 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:03:35.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.234220 22008 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:35.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.234298 22008 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:03:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.242029 22008 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:03:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.242120 22008 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:03:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.242169 22008 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:03:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.242209 22008 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:03:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.242262 22008 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:03:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.242300 22008 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:03:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.242370 22008 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41795/jwks.json
14:03:35.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.251699 22008 master_runner.cc:386] Master server non-default flags:
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.126:34049
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41795/jwks.json
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data/info.pb
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.126
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/logs
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6732
14:03:35.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:03:35.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.253291 22008 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:35.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.255029 22008 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:03:35.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.265035 22013 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:35.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.265075 22014 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:35.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.265136 22016 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:35.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.265960 22008 server_base.cc:1048] running on GCE node
14:03:35.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.266764 22008 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.268064 22008 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:35.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.269300 22008 hybrid_clock.cc:648] HybridClock initialized: now 1750687415269236 us; error 45 us; skew 500 ppm
14:03:35.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.269632 22008 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.276124 22008 webserver.cc:469] Webserver started at http://127.21.123.126:38147/ using document root <none> and password file <none>
14:03:35.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.276919 22008 fs_manager.cc:362] Metadata directory not provided
14:03:35.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.277058 22008 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:35.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.277429 22008 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:03:35.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.280295 22008 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data/instance:
14:03:35.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c51d2fbfbf0a4b38819a021299252b73"
14:03:35.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:35 on dist-test-slave-stbh"
14:03:35.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.281100 22008 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal/instance:
14:03:35.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c51d2fbfbf0a4b38819a021299252b73"
14:03:35.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:35 on dist-test-slave-stbh"
14:03:35.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.286363 22008 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.000s sys 0.007s
14:03:35.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.289584 22022 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.290891 22008 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.291059 22008 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data,/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c51d2fbfbf0a4b38819a021299252b73"
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:35 on dist-test-slave-stbh"
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.291249 22008 fs_report.cc:389] FS layout report
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data/data
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:35.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.305390 22008 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:35.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.306244 22008 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:35.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.306511 22008 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.328394 22008 rpc_server.cc:307] RPC server started. Bound to: 127.21.123.126:34049
14:03:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.328425 22083 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.123.126:34049 every 8 connection(s)
14:03:35.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.329895 22008 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data/info.pb
14:03:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.333317 21997 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu as pid 22008
14:03:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.333526 22084 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.333595 21997 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal/instance
14:03:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.335767 21997 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/wal
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/data
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/logs
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/data/info.pb
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.125:36393
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.125
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41795/jwks.json
14:03:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:03:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:03:35.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.339032 22084 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } has no permanent_uuid. Determining permanent_uuid...
14:03:35.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.353715 22084 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } has no permanent_uuid. Determining permanent_uuid...
14:03:35.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.354882 22026 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111)
14:03:35.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.356321 22084 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.125:36393: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111)
14:03:35.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.405681 22084 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } attempt: 1
14:03:35.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.407549 22084 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.125:36393: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111)
14:03:35.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.475831 22084 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } attempt: 2
14:03:35.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.477670 22084 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.125:36393: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111)
14:03:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.568997 22084 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } attempt: 3
14:03:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.570824 22084 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.125:36393: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111)
14:03:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.647148 22086 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:03:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.647480 22086 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.647593 22086 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:03:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.655604 22086 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:03:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.655695 22086 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:03:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.655745 22086 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:03:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.655784 22086 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:03:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.655838 22086 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:03:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.655879 22086 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:03:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.655921 22086 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41795/jwks.json
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.665073 22086 master_runner.cc:386] Master server non-default flags:
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/data
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/wal
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.125:36393
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41795/jwks.json
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/data/info.pb
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.125
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/logs
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6732
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:03:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.666720 22086 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:35.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.668537 22086 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.678680 22094 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.678776 22095 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.678810 22097 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.679446 22086 server_base.cc:1048] running on GCE node
14:03:35.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.680253 22086 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.681638 22086 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:35.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.682891 22086 hybrid_clock.cc:648] HybridClock initialized: now 1750687415682815 us; error 55 us; skew 500 ppm
14:03:35.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.683239 22086 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:35.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.685441 22086 webserver.cc:469] Webserver started at http://127.21.123.125:43151/ using document root <none> and password file <none>
14:03:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.686194 22086 fs_manager.cc:362] Metadata directory not provided
14:03:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.686314 22086 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.686645 22086 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:03:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.689054 22086 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/data/instance:
14:03:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "891a087227fa4198b42224b537877725"
14:03:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:35 on dist-test-slave-stbh"
14:03:35.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.689898 22086 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/wal/instance:
14:03:35.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "891a087227fa4198b42224b537877725"
14:03:35.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:35 on dist-test-slave-stbh"
14:03:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.694875 22086 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.003s sys 0.001s
14:03:35.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.698179 22103 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:35.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.699401 22086 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
14:03:35.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.699568 22086 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/data,/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/wal
14:03:35.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "891a087227fa4198b42224b537877725"
14:03:35.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:35 on dist-test-slave-stbh"
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.699764 22086 fs_report.cc:389] FS layout report
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/wal
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/wal
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/data/data
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:35.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.714031 22084 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } attempt: 4
14:03:35.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.715847 22084 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.125:36393: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111)
14:03:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.718370 22086 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.719174 22086 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.719446 22086 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:35.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.742044 22086 rpc_server.cc:307] RPC server started. Bound to: 127.21.123.125:36393
14:03:35.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.742127 22164 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.123.125:36393 every 8 connection(s)
14:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.744056 22086 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/data/info.pb
14:03:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.747669 22165 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.752605 21997 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu as pid 22086
14:03:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.752728 21997 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-1/wal/instance
14:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.753557 21997 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal
14:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data
14:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/logs
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data/info.pb
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.124:38761
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.124
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41795/jwks.json
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:03:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.753388 22165 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } has no permanent_uuid. Determining permanent_uuid...
14:03:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.767737 22165 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } has no permanent_uuid. Determining permanent_uuid...
14:03:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.776454 22165 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } has no permanent_uuid. Determining permanent_uuid...
14:03:35.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.777693 22106 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:35.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.779273 22165 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.124:38761: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:35.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.828720 22165 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } attempt: 1
14:03:35.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.830596 22165 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.124:38761: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.898814 22165 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } attempt: 2
14:03:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.900784 22165 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.124:38761: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:35.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:35.992019 22165 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } attempt: 3
14:03:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:35.994118 22165 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.124:38761: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:36.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.015089 22084 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } attempt: 5
14:03:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.021193 22084 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } has no permanent_uuid. Determining permanent_uuid...
14:03:36.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.022879 22084 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.124:38761: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:36.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.065547 22167 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:03:36.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.065913 22167 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:36.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.065996 22167 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:03:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.073505 22167 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:03:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.073593 22167 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:03:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.073638 22167 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:03:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.073675 22167 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:03:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.073722 22167 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:03:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.073789 22167 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:03:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.073840 22167 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41795/jwks.json
14:03:36.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.074116 22084 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } attempt: 1
14:03:36.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.075788 22084 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.124:38761: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.083369 22167 master_runner.cc:386] Master server non-default flags:
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.124:38761
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41795/jwks.json
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data/info.pb
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.124
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/logs
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6732
14:03:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:03:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.084929 22167 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:36.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.086697 22167 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:03:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.096573 22176 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.096674 22178 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.096647 22175 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.097600 22167 server_base.cc:1048] running on GCE node
14:03:36.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.098322 22167 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.099608 22167 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.100850 22167 hybrid_clock.cc:648] HybridClock initialized: now 1750687416100783 us; error 46 us; skew 500 ppm
14:03:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.101162 22167 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:36.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.103260 22167 webserver.cc:469] Webserver started at http://127.21.123.124:45117/ using document root <none> and password file <none>
14:03:36.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.103909 22167 fs_manager.cc:362] Metadata directory not provided
14:03:36.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.104002 22167 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:36.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.104293 22167 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:03:36.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.106617 22167 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data/instance:
14:03:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "289f4dff4a8f4197a18cdb96452c9887"
14:03:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:36 on dist-test-slave-stbh"
14:03:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.107318 22167 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal/instance:
14:03:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "289f4dff4a8f4197a18cdb96452c9887"
14:03:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:36 on dist-test-slave-stbh"
14:03:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.112334 22167 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
14:03:36.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.115533 22184 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:36.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.116842 22167 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.117005 22167 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data,/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "289f4dff4a8f4197a18cdb96452c9887"
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:36 on dist-test-slave-stbh"
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.117201 22167 fs_report.cc:389] FS layout report
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data/data
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:36.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:36.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:36.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:36.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:36.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.131352 22167 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:36.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.132104 22167 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:36.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.132395 22167 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:36.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.137365 22165 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } attempt: 4
14:03:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.139245 22165 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.124:38761: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:36.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.144014 22084 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } attempt: 2
14:03:36.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.145571 22084 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.123.124:38761: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:36.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.154752 22167 rpc_server.cc:307] RPC server started. Bound to: 127.21.123.124:38761
14:03:36.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.154789 22245 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.123.124:38761 every 8 connection(s)
14:03:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.156255 22167 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data/info.pb
14:03:36.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.159880 22246 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:36.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.160737 21997 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu as pid 22167
14:03:36.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.160925 21997 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal/instance
14:03:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.166523 22246 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } has no permanent_uuid. Determining permanent_uuid...
14:03:36.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.179987 22246 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } has no permanent_uuid. Determining permanent_uuid...
14:03:36.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.187307 22246 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } has no permanent_uuid. Determining permanent_uuid...
14:03:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.197809 22246 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: Bootstrap starting.
14:03:36.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.200793 22246 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: Neither blocks nor log segments found. Creating new log.
14:03:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.202112 22246 log.cc:826] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: Log is configured to *not* fsync() on all Append() calls
14:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.204555 22246 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: No bootstrap required, opened a new log
14:03:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.209739 22246 raft_consensus.cc:357] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.210108 22246 raft_consensus.cc:383] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.210182 22246 raft_consensus.cc:738] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 289f4dff4a8f4197a18cdb96452c9887, State: Initialized, Role: FOLLOWER
14:03:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.210778 22246 consensus_queue.cc:260] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.211580 22255 sys_catalog.cc:455] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.211808 22255 sys_catalog.cc:458] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: This master's current role is: FOLLOWER
14:03:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.212591 22246 sys_catalog.cc:564] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: configured and running, proceeding with master startup.
14:03:36.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.222530 22266 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:36.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.222674 22266 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:36.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.220196 22167 master_runner.cc:186] Error getting master registration for 127.21.123.126:34049: OK, instance_id { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" instance_seqno: 1750687415323394 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:03:36.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.225353 22167 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:03:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.251802 22084 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } attempt: 3
14:03:36.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.262108 22084 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: Bootstrap starting.
14:03:36.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.265204 22084 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: Neither blocks nor log segments found. Creating new log.
14:03:36.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.266268 22084 log.cc:826] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: Log is configured to *not* fsync() on all Append() calls
14:03:36.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.268590 22084 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: No bootstrap required, opened a new log
14:03:36.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.273622 22084 raft_consensus.cc:357] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:36.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.273911 22084 raft_consensus.cc:383] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:36.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.273980 22084 raft_consensus.cc:738] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c51d2fbfbf0a4b38819a021299252b73, State: Initialized, Role: FOLLOWER
14:03:36.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.274478 22084 consensus_queue.cc:260] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:36.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.275306 22269 sys_catalog.cc:455] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:36.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.275570 22269 sys_catalog.cc:458] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: This master's current role is: FOLLOWER
14:03:36.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.276289 22084 sys_catalog.cc:564] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: configured and running, proceeding with master startup.
14:03:36.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.285290 22255 raft_consensus.cc:491] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:36.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.285529 22255 raft_consensus.cc:513] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:36.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.285638 22280 catalog_manager.cc:1560] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:36.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.285744 22280 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:36.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.287974 22255 leader_election.cc:290] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049), 891a087227fa4198b42224b537877725 (127.21.123.125:36393)
14:03:36.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.288991 22049 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "289f4dff4a8f4197a18cdb96452c9887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c51d2fbfbf0a4b38819a021299252b73" is_pre_election: true
14:03:36.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.289532 22049 raft_consensus.cc:2466] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 289f4dff4a8f4197a18cdb96452c9887 in term 0.
14:03:36.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.290884 22186 leader_election.cc:304] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 289f4dff4a8f4197a18cdb96452c9887, c51d2fbfbf0a4b38819a021299252b73; no voters:
14:03:36.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.291582 22255 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:03:36.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.291735 22255 raft_consensus.cc:491] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:03:36.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.291831 22255 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 0 FOLLOWER]: Advancing to term 1
14:03:36.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.288120 22008 master_runner.cc:186] Error getting master registration for 127.21.123.125:36393: OK, instance_id { permanent_uuid: "891a087227fa4198b42224b537877725" instance_seqno: 1750687415736727 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:03:36.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.294759 22008 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:03:36.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.295814 22255 raft_consensus.cc:513] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.297113 22049 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "289f4dff4a8f4197a18cdb96452c9887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c51d2fbfbf0a4b38819a021299252b73"
14:03:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.297340 22049 raft_consensus.cc:3058] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 0 FOLLOWER]: Advancing to term 1
14:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.300911 22049 raft_consensus.cc:2466] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 289f4dff4a8f4197a18cdb96452c9887 in term 1.
14:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.301386 22255 leader_election.cc:290] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [CANDIDATE]: Term 1 election: Requested vote from peers c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049), 891a087227fa4198b42224b537877725 (127.21.123.125:36393)
14:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.301395 22186 leader_election.cc:304] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 289f4dff4a8f4197a18cdb96452c9887, c51d2fbfbf0a4b38819a021299252b73; no voters:
14:03:36.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.302913 22255 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 1 FOLLOWER]: Leader election won for term 1
14:03:36.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.303839 22255 raft_consensus.cc:695] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 1 LEADER]: Becoming Leader. State: Replica: 289f4dff4a8f4197a18cdb96452c9887, State: Running, Role: LEADER
14:03:36.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.304538 22255 consensus_queue.cc:237] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:36.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.308426 22282 sys_catalog.cc:455] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 289f4dff4a8f4197a18cdb96452c9887. Latest consensus state: current_term: 1 leader_uuid: "289f4dff4a8f4197a18cdb96452c9887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:36.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.308720 22282 sys_catalog.cc:458] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: This master's current role is: LEADER
14:03:36.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.309089 22028 tablet.cc:2378] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:03:36.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.314301 22284 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:03:36.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.316457 22284 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:03:36.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.324481 22049 raft_consensus.cc:1273] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 1 FOLLOWER]: Refusing update from remote peer 289f4dff4a8f4197a18cdb96452c9887: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:03:36.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.325696 22282 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:03:36.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.353281 22269 sys_catalog.cc:455] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 289f4dff4a8f4197a18cdb96452c9887. Latest consensus state: current_term: 1 leader_uuid: "289f4dff4a8f4197a18cdb96452c9887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:36.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.353636 22269 sys_catalog.cc:458] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: This master's current role is: FOLLOWER
14:03:36.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.361709 22285 mvcc.cc:204] Tried to move back new op lower bound from 7170815657255346176 to 7170815657189490688. Current Snapshot: MvccSnapshot[applied={T|T < 7170815657255346176}]
14:03:36.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.362744 22282 sys_catalog.cc:455] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "289f4dff4a8f4197a18cdb96452c9887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:36.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.362969 22282 sys_catalog.cc:458] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: This master's current role is: LEADER
14:03:36.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.364297 22269 sys_catalog.cc:455] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "289f4dff4a8f4197a18cdb96452c9887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:36.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.364488 22269 sys_catalog.cc:458] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: This master's current role is: FOLLOWER
14:03:36.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.367373 22284 catalog_manager.cc:1349] Generated new cluster ID: e33ca72474a94b3695af9e40ba7dd46c
14:03:36.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.367488 22284 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:03:36.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.390410 22284 catalog_manager.cc:1372] Generated new certificate authority record
14:03:36.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.392839 22284 catalog_manager.cc:1506] Loading token signing keys...
14:03:36.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.407358 22284 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: Generated new TSK 0
14:03:36.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.408456 22284 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.416352 21997 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/wal
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/data
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/logs
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/data/info.pb
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.65:0
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.123.65
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.65
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:03:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:03:36.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.438443 22165 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } attempt: 5
14:03:36.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.451854 22165 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725: Bootstrap starting.
14:03:36.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.456214 22165 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725: Neither blocks nor log segments found. Creating new log.
14:03:36.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.457582 22165 log.cc:826] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725: Log is configured to *not* fsync() on all Append() calls
14:03:36.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.468217 22165 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725: No bootstrap required, opened a new log
14:03:36.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.476586 22165 raft_consensus.cc:357] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:36.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.477178 22165 raft_consensus.cc:383] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:36.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.477429 22165 raft_consensus.cc:738] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 891a087227fa4198b42224b537877725, State: Initialized, Role: FOLLOWER
14:03:36.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.478564 22165 consensus_queue.cc:260] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:36.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.485767 22292 sys_catalog.cc:455] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:36.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.488020 22292 sys_catalog.cc:458] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: This master's current role is: FOLLOWER
14:03:36.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.488211 22130 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "289f4dff4a8f4197a18cdb96452c9887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "891a087227fa4198b42224b537877725" is_pre_election: true
14:03:36.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.488619 22130 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 289f4dff4a8f4197a18cdb96452c9887 in term 0.
14:03:36.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.489264 22128 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 0 FOLLOWER]: Advancing to term 1
14:03:36.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.491779 22128 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 1 FOLLOWER]: Refusing update from remote peer 289f4dff4a8f4197a18cdb96452c9887: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:03:36.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.487927 22129 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "289f4dff4a8f4197a18cdb96452c9887" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "891a087227fa4198b42224b537877725"
14:03:36.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.493307 22255 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [LEADER]: Connected to new peer: Peer: permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:03:36.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.487874 22165 sys_catalog.cc:564] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: configured and running, proceeding with master startup.
14:03:36.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.495675 22129 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 289f4dff4a8f4197a18cdb96452c9887 in term 1.
14:03:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.544165 22292 sys_catalog.cc:455] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 289f4dff4a8f4197a18cdb96452c9887. Latest consensus state: current_term: 1 leader_uuid: "289f4dff4a8f4197a18cdb96452c9887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.544499 22292 sys_catalog.cc:458] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: This master's current role is: FOLLOWER
14:03:36.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.545223 22299 sys_catalog.cc:455] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "289f4dff4a8f4197a18cdb96452c9887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:36.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.545706 22299 sys_catalog.cc:458] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: This master's current role is: FOLLOWER
14:03:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.568010 22309 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:03:36.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.581097 22255 sys_catalog.cc:455] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "289f4dff4a8f4197a18cdb96452c9887" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:36.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.581424 22255 sys_catalog.cc:458] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: This master's current role is: LEADER
14:03:36.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.858999 22290 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:03:36.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.859328 22290 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:36.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.859479 22290 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:03:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.868357 22290 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:03:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.868539 22290 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:03:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.868611 22290 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:03:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.868669 22290 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:03:36.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.868808 22290 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.123.65
14:03:36.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.879395 22290 tablet_server_runner.cc:78] Tablet server non-default flags:
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/data
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/wal
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.65:0
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/data/info.pb
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.65
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.123.65
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/logs
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6732
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.881052 22290 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.882879 22290 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:03:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.894500 22315 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.894480 22314 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:36.894623 22317 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:36.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.895548 22290 server_base.cc:1048] running on GCE node
14:03:36.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.896133 22290 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:36.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.896971 22290 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.898211 22290 hybrid_clock.cc:648] HybridClock initialized: now 1750687416898142 us; error 36 us; skew 500 ppm
14:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.898470 22290 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:36.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.900606 22290 webserver.cc:469] Webserver started at http://127.21.123.65:39365/ using document root <none> and password file <none>
14:03:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.901242 22290 fs_manager.cc:362] Metadata directory not provided
14:03:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.901340 22290 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.901613 22290 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:03:36.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.904049 22290 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/data/instance:
14:03:36.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e6afa1c1602247b682b32d093471e18d"
14:03:36.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:36 on dist-test-slave-stbh"
14:03:36.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.904788 22290 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/wal/instance:
14:03:36.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e6afa1c1602247b682b32d093471e18d"
14:03:36.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:36 on dist-test-slave-stbh"
14:03:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.909770 22290 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
14:03:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.913122 22323 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.914510 22290 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:03:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.914638 22290 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/data,/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/wal
14:03:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e6afa1c1602247b682b32d093471e18d"
14:03:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:36 on dist-test-slave-stbh"
14:03:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.914791 22290 fs_report.cc:389] FS layout report
14:03:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/wal
14:03:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/wal
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/data/data
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:36.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.932099 22290 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:36.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.932839 22290 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:36.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.933094 22290 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.935940 22290 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:03:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.936072 22290 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:03:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.936195 22290 ts_tablet_manager.cc:610] Registered 0 tablets
14:03:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.936280 22290 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:03:36.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.999255 22290 rpc_server.cc:307] RPC server started. Bound to: 127.21.123.65:45201
14:03:36.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:36.999312 22447 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.123.65:45201 every 8 connection(s)
14:03:37.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.000626 22290 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/data/info.pb
14:03:37.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.004588 21997 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu as pid 22290
14:03:37.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.004778 21997 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-0/wal/instance
14:03:37.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.003832 22449 heartbeater.cc:344] Connected to a master server at 127.21.123.126:34049
14:03:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.008253 22449 heartbeater.cc:461] Registering TS with master...
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.009066 21997 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/wal
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/data
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/logs
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/data/info.pb
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.66:0
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.123.66
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.66
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:03:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:03:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:03:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.009497 22449 heartbeater.cc:507] Master 127.21.123.126:34049 requested a full tablet report, sending...
14:03:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.010105 22448 heartbeater.cc:344] Connected to a master server at 127.21.123.124:38761
14:03:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.010246 22448 heartbeater.cc:461] Registering TS with master...
14:03:37.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.010871 22448 heartbeater.cc:507] Master 127.21.123.124:38761 requested a full tablet report, sending...
14:03:37.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.012079 22039 ts_manager.cc:194] Registered new tserver with Master: e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201)
14:03:37.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.012461 22201 ts_manager.cc:194] Registered new tserver with Master: e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201)
14:03:37.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.014519 22201 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.123.65:36503
14:03:37.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.016722 22450 heartbeater.cc:344] Connected to a master server at 127.21.123.125:36393
14:03:37.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.016880 22450 heartbeater.cc:461] Registering TS with master...
14:03:37.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.017186 22450 heartbeater.cc:507] Master 127.21.123.125:36393 requested a full tablet report, sending...
14:03:37.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.018558 22120 ts_manager.cc:194] Registered new tserver with Master: e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201)
14:03:37.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.287628 22280 catalog_manager.cc:1261] Loaded cluster ID: e33ca72474a94b3695af9e40ba7dd46c
14:03:37.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.287731 22280 catalog_manager.cc:1554] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: loading cluster ID for follower catalog manager: success
14:03:37.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.290400 22280 catalog_manager.cc:1576] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: acquiring CA information for follower catalog manager: success
14:03:37.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.292147 22280 catalog_manager.cc:1604] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:03:37.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.327075 22453 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:03:37.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.327360 22453 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:37.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.327453 22453 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:03:37.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.334981 22453 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:03:37.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.335163 22453 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:03:37.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.335245 22453 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:03:37.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.335302 22453 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:03:37.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.335453 22453 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.123.66
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.344518 22453 tablet_server_runner.cc:78] Tablet server non-default flags:
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/data
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/wal
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.66:0
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/data/info.pb
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.66
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.123.66
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/logs
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6732
14:03:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:03:37.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.346115 22453 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:37.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.347820 22453 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:03:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.359090 22461 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.359123 22458 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.359084 22459 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.360055 22453 server_base.cc:1048] running on GCE node
14:03:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.360576 22453 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:37.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.361405 22453 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:37.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.362630 22453 hybrid_clock.cc:648] HybridClock initialized: now 1750687417362563 us; error 39 us; skew 500 ppm
14:03:37.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.362910 22453 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:37.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.365113 22453 webserver.cc:469] Webserver started at http://127.21.123.66:34143/ using document root <none> and password file <none>
14:03:37.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.365832 22453 fs_manager.cc:362] Metadata directory not provided
14:03:37.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.365967 22453 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:37.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.366286 22453 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:03:37.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.368571 22453 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/data/instance:
14:03:37.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3"
14:03:37.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:37 on dist-test-slave-stbh"
14:03:37.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.369307 22453 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/wal/instance:
14:03:37.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3"
14:03:37.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:37 on dist-test-slave-stbh"
14:03:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.374310 22453 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
14:03:37.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.377297 22467 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:37.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.378521 22453 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
14:03:37.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.378674 22453 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/data,/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/wal
14:03:37.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3"
14:03:37.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:37 on dist-test-slave-stbh"
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.378871 22453 fs_report.cc:389] FS layout report
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/wal
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/wal
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/data/data
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:37.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.396068 22453 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:37.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.396924 22453 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:37.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.397253 22453 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.400660 22453 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:03:37.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.400899 22453 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
14:03:37.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.401046 22453 ts_tablet_manager.cc:610] Registered 0 tablets
14:03:37.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.401129 22453 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:03:37.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.468250 22453 rpc_server.cc:307] RPC server started. Bound to: 127.21.123.66:45807
14:03:37.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.468290 22588 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.123.66:45807 every 8 connection(s)
14:03:37.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.469668 22453 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/data/info.pb
14:03:37.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.473531 22590 heartbeater.cc:344] Connected to a master server at 127.21.123.126:34049
14:03:37.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.473944 22590 heartbeater.cc:461] Registering TS with master...
14:03:37.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.475067 22590 heartbeater.cc:507] Master 127.21.123.126:34049 requested a full tablet report, sending...
14:03:37.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.476680 22039 ts_manager.cc:194] Registered new tserver with Master: 8c899ae4122d4a4fa4aa06b02a2ca2d3 (127.21.123.66:45807)
14:03:37.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.478260 21997 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu as pid 22453
14:03:37.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.478446 21997 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-1/wal/instance
14:03:37.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.479867 22589 heartbeater.cc:344] Connected to a master server at 127.21.123.124:38761
14:03:37.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.479992 22589 heartbeater.cc:461] Registering TS with master...
14:03:37.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.480461 22589 heartbeater.cc:507] Master 127.21.123.124:38761 requested a full tablet report, sending...
14:03:37.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.481431 22200 ts_manager.cc:194] Registered new tserver with Master: 8c899ae4122d4a4fa4aa06b02a2ca2d3 (127.21.123.66:45807)
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.482561 22200 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.123.66:46123
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.482683 21997 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/wal
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/data
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/logs
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/data/info.pb
14:03:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.67:0
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.123.67
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.67
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:03:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:03:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.486892 22591 heartbeater.cc:344] Connected to a master server at 127.21.123.125:36393
14:03:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.487143 22591 heartbeater.cc:461] Registering TS with master...
14:03:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.487473 22591 heartbeater.cc:507] Master 127.21.123.125:36393 requested a full tablet report, sending...
14:03:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.488327 22120 ts_manager.cc:194] Registered new tserver with Master: 8c899ae4122d4a4fa4aa06b02a2ca2d3 (127.21.123.66:45807)
14:03:37.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.573684 22309 catalog_manager.cc:1261] Loaded cluster ID: e33ca72474a94b3695af9e40ba7dd46c
14:03:37.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.573860 22309 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725: loading cluster ID for follower catalog manager: success
14:03:37.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.577961 22309 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725: acquiring CA information for follower catalog manager: success
14:03:37.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.580294 22309 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:03:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.815912 22594 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:03:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.816265 22594 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.816371 22594 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:03:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.824889 22594 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:03:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.825024 22594 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:03:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.825073 22594 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:03:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.825106 22594 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:03:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.825201 22594 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.123.67
14:03:37.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.834527 22594 tablet_server_runner.cc:78] Tablet server non-default flags:
14:03:37.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:37.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:37.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/data
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/wal
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.67:0
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/data/info.pb
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.67
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.123.67
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/logs
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6732
14:03:37.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:03:37.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.836064 22594 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:37.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.837819 22594 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:03:37.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.850183 22601 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:37.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.850284 22600 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:37.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:37.851178 22603 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:37.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.851519 22594 server_base.cc:1048] running on GCE node
14:03:37.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.852291 22594 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:37.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.853469 22594 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:37.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.854954 22594 hybrid_clock.cc:648] HybridClock initialized: now 1750687417854889 us; error 30 us; skew 500 ppm
14:03:37.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.855319 22594 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.858196 22594 webserver.cc:469] Webserver started at http://127.21.123.67:44491/ using document root <none> and password file <none>
14:03:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.859194 22594 fs_manager.cc:362] Metadata directory not provided
14:03:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.859346 22594 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.859740 22594 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:03:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.863205 22594 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/data/instance:
14:03:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0ed313e058b5467f883ba31f1d164b3d"
14:03:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:37 on dist-test-slave-stbh"
14:03:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.864251 22594 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/wal/instance:
14:03:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0ed313e058b5467f883ba31f1d164b3d"
14:03:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:37 on dist-test-slave-stbh"
14:03:37.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.871038 22594 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.003s sys 0.005s
14:03:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.875291 22609 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.876984 22594 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.877184 22594 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/data,/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/wal
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0ed313e058b5467f883ba31f1d164b3d"
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:37 on dist-test-slave-stbh"
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.877349 22594 fs_report.cc:389] FS layout report
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/wal
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/wal
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/data/data
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:37.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.899475 22594 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:37.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.900589 22594 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:37.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.900959 22594 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:37.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.904708 22594 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:03:37.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.904826 22594 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:03:37.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.904937 22594 ts_tablet_manager.cc:610] Registered 0 tablets
14:03:37.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.905016 22594 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:03:37.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.991899 22594 rpc_server.cc:307] RPC server started. Bound to: 127.21.123.67:44431
14:03:37.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.991905 22730 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.123.67:44431 every 8 connection(s)
14:03:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:37.993556 22594 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/data/info.pb
14:03:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.002215 21997 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu as pid 22594
14:03:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.002391 21997 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/ts-2/wal/instance
14:03:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.010226 22732 heartbeater.cc:344] Connected to a master server at 127.21.123.126:34049
14:03:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.010574 22732 heartbeater.cc:461] Registering TS with master...
14:03:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.011600 22732 heartbeater.cc:507] Master 127.21.123.126:34049 requested a full tablet report, sending...
14:03:38.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.013253 22039 ts_manager.cc:194] Registered new tserver with Master: 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67:44431)
14:03:38.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.015012 22733 heartbeater.cc:344] Connected to a master server at 127.21.123.125:36393
14:03:38.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.015151 22733 heartbeater.cc:461] Registering TS with master...
14:03:38.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.015765 22733 heartbeater.cc:507] Master 127.21.123.125:36393 requested a full tablet report, sending...
14:03:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.017019 22120 ts_manager.cc:194] Registered new tserver with Master: 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67:44431)
14:03:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.017822 22731 heartbeater.cc:344] Connected to a master server at 127.21.123.124:38761
14:03:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.017807 22448 heartbeater.cc:499] Master 127.21.123.124:38761 was elected leader, sending a full tablet report...
14:03:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.017946 22731 heartbeater.cc:461] Registering TS with master...
14:03:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.018335 22731 heartbeater.cc:507] Master 127.21.123.124:38761 requested a full tablet report, sending...
14:03:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.019220 22201 ts_manager.cc:194] Registered new tserver with Master: 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67:44431)
14:03:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.020082 22201 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.123.67:54759
14:03:38.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.027773 21997 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
14:03:38.029 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:03:38.029 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:03:38.030 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.21.123.126"
port: 34049
}
bound_http_address {
host: "127.21.123.126"
port: 38147
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.21.123.125"
port: 36393
}
bound_http_address {
host: "127.21.123.125"
port: 43151
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.21.123.124"
port: 38761
}
bound_http_address {
host: "127.21.123.124"
port: 45117
}
}
}
14:03:38.031 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:03:38.031 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.21.123.65"
port: 45201
}
bound_http_address {
host: "127.21.123.65"
port: 39365
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.21.123.66"
port: 45807
}
bound_http_address {
host: "127.21.123.66"
port: 34143
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.21.123.67"
port: 44431
}
bound_http_address {
host: "127.21.123.67"
port: 44491
}
}
}
14:03:38.031 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:03:38.033 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:03:38.034 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.123.125:36393(127.21.123.125:36393)] connecting to peer
14:03:38.035 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:38.036 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.123.126:34049(127.21.123.126:34049)] connecting to peer
14:03:38.037 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.123.125:36393(127.21.123.125:36393)] Successfully connected to peer
14:03:38.037 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.123.124:38761(127.21.123.124:38761)] Successfully connected to peer
14:03:38.038 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.123.126:34049(127.21.123.126:34049)] Successfully connected to peer
14:03:38.039 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.123.125
14:03:38.040 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.123.124
14:03:38.052 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.123.126
14:03:38.055 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7a5588e5, L:/127.0.0.1:40342 - R:/127.21.123.124:38761] using SASL/PLAIN
14:03:38.058 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd2e415e5, L:/127.0.0.1:45268 - R:/127.21.123.125:36393] using SASL/PLAIN
14:03:38.058 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:03:38.058 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=24), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [23ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: OK, deferred=null) after lookup
14:03:38.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.060372 22201 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40342:
14:03:38.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
14:03:38.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:03:38.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:38.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
14:03:38.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:03:38.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:38.062695 22201 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:03:38.067 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3584d195, L:/127.0.0.1:47444 - R:/127.21.123.126:34049] using SASL/PLAIN
14:03:38.085 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 08d3968b837044f989a5b012b478ceae
14:03:38.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.126173 22523 tablet_service.cc:1468] Processing CreateTablet for tablet 79e2cea363a6408d9a53d72158f80fa2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:03:38.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.127009 22665 tablet_service.cc:1468] Processing CreateTablet for tablet 79e2cea363a6408d9a53d72158f80fa2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:03:38.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.126155 22524 tablet_service.cc:1468] Processing CreateTablet for tablet a382527694d540cb9a48ab998871f9ff (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:03:38.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.126838 22666 tablet_service.cc:1468] Processing CreateTablet for tablet a382527694d540cb9a48ab998871f9ff (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:03:38.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.127867 22523 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 79e2cea363a6408d9a53d72158f80fa2. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:38.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.128258 22524 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a382527694d540cb9a48ab998871f9ff. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:38.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.128461 22666 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a382527694d540cb9a48ab998871f9ff. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:38.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.128829 22665 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 79e2cea363a6408d9a53d72158f80fa2. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:38.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.135177 22377 tablet_service.cc:1468] Processing CreateTablet for tablet 79e2cea363a6408d9a53d72158f80fa2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:03:38.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.136592 22377 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 79e2cea363a6408d9a53d72158f80fa2. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:38.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.137728 22378 tablet_service.cc:1468] Processing CreateTablet for tablet a382527694d540cb9a48ab998871f9ff (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:03:38.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.138257 22378 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a382527694d540cb9a48ab998871f9ff. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:38.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.147017 22757 tablet_bootstrap.cc:492] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Bootstrap starting.
14:03:38.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.150413 22758 tablet_bootstrap.cc:492] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d: Bootstrap starting.
14:03:38.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.150792 22757 tablet_bootstrap.cc:654] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Neither blocks nor log segments found. Creating new log.
14:03:38.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.152063 22759 tablet_bootstrap.cc:492] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d: Bootstrap starting.
14:03:38.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.152331 22757 log.cc:826] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Log is configured to *not* fsync() on all Append() calls
14:03:38.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.154453 22758 tablet_bootstrap.cc:654] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d: Neither blocks nor log segments found. Creating new log.
14:03:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.155467 22757 tablet_bootstrap.cc:492] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: No bootstrap required, opened a new log
14:03:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.155682 22758 log.cc:826] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d: Log is configured to *not* fsync() on all Append() calls
14:03:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.155755 22757 ts_tablet_manager.cc:1397] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.005s
14:03:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.157220 22759 tablet_bootstrap.cc:654] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d: Neither blocks nor log segments found. Creating new log.
14:03:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.158480 22759 log.cc:826] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d: Log is configured to *not* fsync() on all Append() calls
14:03:38.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.164448 22758 tablet_bootstrap.cc:492] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d: No bootstrap required, opened a new log
14:03:38.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.164832 22758 ts_tablet_manager.cc:1397] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.010s
14:03:38.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.164567 22757 raft_consensus.cc:357] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.165027 22757 raft_consensus.cc:383] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:38.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.165361 22757 raft_consensus.cc:738] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c899ae4122d4a4fa4aa06b02a2ca2d3, State: Initialized, Role: FOLLOWER
14:03:38.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.166431 22757 consensus_queue.cc:260] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.171720 22758 raft_consensus.cc:357] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.172096 22758 raft_consensus.cc:383] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:38.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.172185 22758 raft_consensus.cc:738] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0ed313e058b5467f883ba31f1d164b3d, State: Initialized, Role: FOLLOWER
14:03:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.172788 22758 consensus_queue.cc:260] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.173861 22731 heartbeater.cc:499] Master 127.21.123.124:38761 was elected leader, sending a full tablet report...
14:03:38.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.175812 22589 heartbeater.cc:499] Master 127.21.123.124:38761 was elected leader, sending a full tablet report...
14:03:38.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.178118 22757 ts_tablet_manager.cc:1428] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Time spent starting tablet: real 0.022s user 0.019s sys 0.000s
14:03:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.180106 22757 tablet_bootstrap.cc:492] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Bootstrap starting.
14:03:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.181655 22759 tablet_bootstrap.cc:492] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d: No bootstrap required, opened a new log
14:03:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.182027 22759 ts_tablet_manager.cc:1397] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d: Time spent bootstrapping tablet: real 0.030s user 0.019s sys 0.003s
14:03:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.182651 22757 tablet_bootstrap.cc:654] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Neither blocks nor log segments found. Creating new log.
14:03:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.184650 22758 ts_tablet_manager.cc:1428] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d: Time spent starting tablet: real 0.020s user 0.008s sys 0.001s
14:03:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.185078 22758 tablet_bootstrap.cc:492] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d: Bootstrap starting.
14:03:38.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.186757 22758 tablet_bootstrap.cc:654] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d: Neither blocks nor log segments found. Creating new log.
14:03:38.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.189884 22757 tablet_bootstrap.cc:492] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3: No bootstrap required, opened a new log
14:03:38.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.190047 22757 ts_tablet_manager.cc:1397] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Time spent bootstrapping tablet: real 0.010s user 0.008s sys 0.000s
14:03:38.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.190104 22758 tablet_bootstrap.cc:492] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d: No bootstrap required, opened a new log
14:03:38.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.190263 22758 ts_tablet_manager.cc:1397] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.003s
14:03:38.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.190788 22757 raft_consensus.cc:357] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } }
14:03:38.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.190925 22758 raft_consensus.cc:357] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } }
14:03:38.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.191129 22758 raft_consensus.cc:383] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:38.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.191233 22758 raft_consensus.cc:738] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0ed313e058b5467f883ba31f1d164b3d, State: Initialized, Role: FOLLOWER
14:03:38.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.191184 22757 raft_consensus.cc:383] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:38.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.191442 22757 raft_consensus.cc:738] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c899ae4122d4a4fa4aa06b02a2ca2d3, State: Initialized, Role: FOLLOWER
14:03:38.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.191476 22758 consensus_queue.cc:260] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } }
14:03:38.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.191807 22757 consensus_queue.cc:260] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } }
14:03:38.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.191934 22758 ts_tablet_manager.cc:1428] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
14:03:38.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.192448 22757 ts_tablet_manager.cc:1428] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
14:03:38.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.196429 22759 raft_consensus.cc:357] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.196985 22759 raft_consensus.cc:383] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:38.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.197139 22759 raft_consensus.cc:738] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e6afa1c1602247b682b32d093471e18d, State: Initialized, Role: FOLLOWER
14:03:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.197763 22759 consensus_queue.cc:260] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.199328 22759 ts_tablet_manager.cc:1428] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d: Time spent starting tablet: real 0.017s user 0.006s sys 0.003s
14:03:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.199676 22759 tablet_bootstrap.cc:492] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d: Bootstrap starting.
14:03:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.201511 22759 tablet_bootstrap.cc:654] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d: Neither blocks nor log segments found. Creating new log.
14:03:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.203642 22759 tablet_bootstrap.cc:492] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d: No bootstrap required, opened a new log
14:03:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.203790 22759 ts_tablet_manager.cc:1397] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
14:03:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.204412 22759 raft_consensus.cc:357] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } }
14:03:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.204564 22759 raft_consensus.cc:383] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.204638 22759 raft_consensus.cc:738] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e6afa1c1602247b682b32d093471e18d, State: Initialized, Role: FOLLOWER
14:03:38.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.204861 22759 consensus_queue.cc:260] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } }
14:03:38.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.205353 22759 ts_tablet_manager.cc:1428] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d: Time spent starting tablet: real 0.001s user 0.001s sys 0.001s
14:03:38.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:38.222733 22592 tablet.cc:2378] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:03:38.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.228942 22764 raft_consensus.cc:491] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:38.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.229179 22764 raft_consensus.cc:513] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } }
14:03:38.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.230968 22764 leader_election.cc:290] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8c899ae4122d4a4fa4aa06b02a2ca2d3 (127.21.123.66:45807), e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201)
14:03:38.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.234656 22763 raft_consensus.cc:491] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:38.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.234970 22763 raft_consensus.cc:513] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } }
14:03:38.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.237548 22763 leader_election.cc:290] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67:44431), e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201)
14:03:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.238407 22403 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a382527694d540cb9a48ab998871f9ff" candidate_uuid: "0ed313e058b5467f883ba31f1d164b3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e6afa1c1602247b682b32d093471e18d" is_pre_election: true
14:03:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.238798 22544 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a382527694d540cb9a48ab998871f9ff" candidate_uuid: "0ed313e058b5467f883ba31f1d164b3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" is_pre_election: true
14:03:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.238962 22403 raft_consensus.cc:2466] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0ed313e058b5467f883ba31f1d164b3d in term 0.
14:03:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.239190 22544 raft_consensus.cc:2466] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0ed313e058b5467f883ba31f1d164b3d in term 0.
14:03:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.239647 22610 leader_election.cc:304] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0ed313e058b5467f883ba31f1d164b3d, e6afa1c1602247b682b32d093471e18d; no voters:
14:03:38.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.240257 22764 raft_consensus.cc:2802] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Leader pre-election won for term 1
14:03:38.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.240448 22764 raft_consensus.cc:491] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:03:38.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.240568 22764 raft_consensus.cc:3058] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Advancing to term 1
14:03:38.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.243762 22764 raft_consensus.cc:513] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } }
14:03:38.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.244635 22764 leader_election.cc:290] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [CANDIDATE]: Term 1 election: Requested vote from peers 8c899ae4122d4a4fa4aa06b02a2ca2d3 (127.21.123.66:45807), e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201)
14:03:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.244915 22544 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a382527694d540cb9a48ab998871f9ff" candidate_uuid: "0ed313e058b5467f883ba31f1d164b3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3"
14:03:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.244956 22403 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a382527694d540cb9a48ab998871f9ff" candidate_uuid: "0ed313e058b5467f883ba31f1d164b3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e6afa1c1602247b682b32d093471e18d"
14:03:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.245105 22544 raft_consensus.cc:3058] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Advancing to term 1
14:03:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.245143 22403 raft_consensus.cc:3058] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Advancing to term 1
14:03:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.247727 22403 raft_consensus.cc:2466] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0ed313e058b5467f883ba31f1d164b3d in term 1.
14:03:38.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.248124 22610 leader_election.cc:304] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0ed313e058b5467f883ba31f1d164b3d, e6afa1c1602247b682b32d093471e18d; no voters:
14:03:38.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.248255 22544 raft_consensus.cc:2466] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0ed313e058b5467f883ba31f1d164b3d in term 1.
14:03:38.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.248705 22764 raft_consensus.cc:2802] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [term 1 FOLLOWER]: Leader election won for term 1
14:03:38.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.248775 22686 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a382527694d540cb9a48ab998871f9ff" candidate_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0ed313e058b5467f883ba31f1d164b3d" is_pre_election: true
14:03:38.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.251266 22764 raft_consensus.cc:695] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [term 1 LEADER]: Becoming Leader. State: Replica: 0ed313e058b5467f883ba31f1d164b3d, State: Running, Role: LEADER
14:03:38.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:38.251924 22734 tablet.cc:2378] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:03:38.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.251870 22764 consensus_queue.cc:237] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } }
14:03:38.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.253216 22403 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a382527694d540cb9a48ab998871f9ff" candidate_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e6afa1c1602247b682b32d093471e18d" is_pre_election: true
14:03:38.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.253535 22403 raft_consensus.cc:2391] T a382527694d540cb9a48ab998871f9ff P e6afa1c1602247b682b32d093471e18d [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 8c899ae4122d4a4fa4aa06b02a2ca2d3 in current term 1: Already voted for candidate 0ed313e058b5467f883ba31f1d164b3d in this term.
14:03:38.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:38.256503 22451 tablet.cc:2378] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:03:38.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.257185 22200 catalog_manager.cc:5582] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d reported cstate change: term changed from 0 to 1, leader changed from <none> to 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67). New cstate: current_term: 1 leader_uuid: "0ed313e058b5467f883ba31f1d164b3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } health_report { overall_health: UNKNOWN } } }
14:03:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.257946 22468 leader_election.cc:304] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8c899ae4122d4a4fa4aa06b02a2ca2d3; no voters: 0ed313e058b5467f883ba31f1d164b3d, e6afa1c1602247b682b32d093471e18d
14:03:38.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.258463 22763 raft_consensus.cc:2747] T a382527694d540cb9a48ab998871f9ff P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:03:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.341889 22763 raft_consensus.cc:491] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.342073 22763 raft_consensus.cc:513] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.342625 22763 leader_election.cc:290] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201), 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67:44431)
14:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.343300 22686 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79e2cea363a6408d9a53d72158f80fa2" candidate_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0ed313e058b5467f883ba31f1d164b3d" is_pre_election: true
14:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.343297 22403 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79e2cea363a6408d9a53d72158f80fa2" candidate_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e6afa1c1602247b682b32d093471e18d" is_pre_election: true
14:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.343542 22403 raft_consensus.cc:2466] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8c899ae4122d4a4fa4aa06b02a2ca2d3 in term 0.
14:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.343562 22686 raft_consensus.cc:2466] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8c899ae4122d4a4fa4aa06b02a2ca2d3 in term 0.
14:03:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.344010 22468 leader_election.cc:304] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0ed313e058b5467f883ba31f1d164b3d, 8c899ae4122d4a4fa4aa06b02a2ca2d3; no voters:
14:03:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.344270 22763 raft_consensus.cc:2802] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:03:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.344390 22763 raft_consensus.cc:491] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:03:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.344496 22763 raft_consensus.cc:3058] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Advancing to term 1
14:03:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.346496 22763 raft_consensus.cc:513] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.347025 22763 leader_election.cc:290] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [CANDIDATE]: Term 1 election: Requested vote from peers e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201), 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67:44431)
14:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.347442 22403 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79e2cea363a6408d9a53d72158f80fa2" candidate_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e6afa1c1602247b682b32d093471e18d"
14:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.347494 22686 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "79e2cea363a6408d9a53d72158f80fa2" candidate_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0ed313e058b5467f883ba31f1d164b3d"
14:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.347606 22403 raft_consensus.cc:3058] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Advancing to term 1
14:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.347687 22686 raft_consensus.cc:3058] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Advancing to term 1
14:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.349395 22403 raft_consensus.cc:2466] T 79e2cea363a6408d9a53d72158f80fa2 P e6afa1c1602247b682b32d093471e18d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8c899ae4122d4a4fa4aa06b02a2ca2d3 in term 1.
14:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.349700 22686 raft_consensus.cc:2466] T 79e2cea363a6408d9a53d72158f80fa2 P 0ed313e058b5467f883ba31f1d164b3d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8c899ae4122d4a4fa4aa06b02a2ca2d3 in term 1.
14:03:38.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.349853 22468 leader_election.cc:304] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8c899ae4122d4a4fa4aa06b02a2ca2d3, e6afa1c1602247b682b32d093471e18d; no voters:
14:03:38.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.350170 22763 raft_consensus.cc:2802] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 1 FOLLOWER]: Leader election won for term 1
14:03:38.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.350622 22763 raft_consensus.cc:695] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 1 LEADER]: Becoming Leader. State: Replica: 8c899ae4122d4a4fa4aa06b02a2ca2d3, State: Running, Role: LEADER
14:03:38.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.351385 22763 consensus_queue.cc:237] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.356453 22200 catalog_manager.cc:5582] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8c899ae4122d4a4fa4aa06b02a2ca2d3 (127.21.123.66). New cstate: current_term: 1 leader_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } health_report { overall_health: UNKNOWN } } }
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.454164 22200 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40354:
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:03:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:38.455044 22200 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:03:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.466354 22523 tablet_service.cc:1468] Processing CreateTablet for tablet 4bd3f6916eba4f2fb1a9aace41281511 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=71e4144517084e3087907b27d7a03a66]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:03:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.466588 22377 tablet_service.cc:1468] Processing CreateTablet for tablet 4bd3f6916eba4f2fb1a9aace41281511 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=71e4144517084e3087907b27d7a03a66]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:03:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.466596 22665 tablet_service.cc:1468] Processing CreateTablet for tablet 4bd3f6916eba4f2fb1a9aace41281511 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=71e4144517084e3087907b27d7a03a66]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:03:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.467307 22523 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4bd3f6916eba4f2fb1a9aace41281511. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.467419 22665 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4bd3f6916eba4f2fb1a9aace41281511. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.467420 22377 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4bd3f6916eba4f2fb1a9aace41281511. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.474380 22758 tablet_bootstrap.cc:492] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: Bootstrap starting.
14:03:38.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.475611 22757 tablet_bootstrap.cc:492] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Bootstrap starting.
14:03:38.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.476039 22758 tablet_bootstrap.cc:654] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: Neither blocks nor log segments found. Creating new log.
14:03:38.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.477558 22757 tablet_bootstrap.cc:654] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Neither blocks nor log segments found. Creating new log.
14:03:38.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.479784 22759 tablet_bootstrap.cc:492] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d: Bootstrap starting.
14:03:38.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.481451 22759 tablet_bootstrap.cc:654] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d: Neither blocks nor log segments found. Creating new log.
14:03:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.486960 22758 tablet_bootstrap.cc:492] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: No bootstrap required, opened a new log
14:03:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.487041 22759 tablet_bootstrap.cc:492] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d: No bootstrap required, opened a new log
14:03:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.487135 22758 ts_tablet_manager.cc:1397] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.007s
14:03:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.487193 22759 ts_tablet_manager.cc:1397] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.004s
14:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.487912 22759 raft_consensus.cc:357] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.487944 22758 raft_consensus.cc:357] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.488111 22758 raft_consensus.cc:383] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.488111 22759 raft_consensus.cc:383] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.488195 22758 raft_consensus.cc:738] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0ed313e058b5467f883ba31f1d164b3d, State: Initialized, Role: FOLLOWER
14:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.488210 22759 raft_consensus.cc:738] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e6afa1c1602247b682b32d093471e18d, State: Initialized, Role: FOLLOWER
14:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.488480 22758 consensus_queue.cc:260] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.488438 22759 consensus_queue.cc:260] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.488958 22759 ts_tablet_manager.cc:1428] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
14:03:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.489004 22758 ts_tablet_manager.cc:1428] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
14:03:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.488914 22764 tablet_replica.cc:440] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } }
14:03:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.489148 22764 tablet_replica.cc:443] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: This TxnStatusTablet replica's current role is: FOLLOWER
14:03:38.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.489485 22767 tablet_replica.cc:440] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } }
14:03:38.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.489800 22767 tablet_replica.cc:443] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d: This TxnStatusTablet replica's current role is: FOLLOWER
14:03:38.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.490356 22757 tablet_bootstrap.cc:492] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: No bootstrap required, opened a new log
14:03:38.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.490625 22757 ts_tablet_manager.cc:1397] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Time spent bootstrapping tablet: real 0.015s user 0.007s sys 0.004s
14:03:38.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.491529 22757 raft_consensus.cc:357] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.491709 22757 raft_consensus.cc:383] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:38.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.491792 22757 raft_consensus.cc:738] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c899ae4122d4a4fa4aa06b02a2ca2d3, State: Initialized, Role: FOLLOWER
14:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.492039 22757 consensus_queue.cc:260] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.492579 22763 tablet_replica.cc:440] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } }
14:03:38.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.492823 22763 tablet_replica.cc:443] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: This TxnStatusTablet replica's current role is: FOLLOWER
14:03:38.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.494558 22757 ts_tablet_manager.cc:1428] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
14:03:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.596320 22764 raft_consensus.cc:491] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.596571 22764 raft_consensus.cc:513] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.597105 22764 leader_election.cc:290] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201), 8c899ae4122d4a4fa4aa06b02a2ca2d3 (127.21.123.66:45807)
14:03:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.597594 22403 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4bd3f6916eba4f2fb1a9aace41281511" candidate_uuid: "0ed313e058b5467f883ba31f1d164b3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e6afa1c1602247b682b32d093471e18d" is_pre_election: true
14:03:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.597659 22544 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4bd3f6916eba4f2fb1a9aace41281511" candidate_uuid: "0ed313e058b5467f883ba31f1d164b3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" is_pre_election: true
14:03:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.597796 22403 raft_consensus.cc:2466] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0ed313e058b5467f883ba31f1d164b3d in term 0.
14:03:38.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.597955 22544 raft_consensus.cc:2466] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0ed313e058b5467f883ba31f1d164b3d in term 0.
14:03:38.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.598189 22610 leader_election.cc:304] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0ed313e058b5467f883ba31f1d164b3d, e6afa1c1602247b682b32d093471e18d; no voters:
14:03:38.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.598521 22764 raft_consensus.cc:2802] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Leader pre-election won for term 1
14:03:38.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.598618 22764 raft_consensus.cc:491] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:03:38.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.598695 22764 raft_consensus.cc:3058] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [term 0 FOLLOWER]: Advancing to term 1
14:03:38.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.600441 22764 raft_consensus.cc:513] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.600943 22764 leader_election.cc:290] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [CANDIDATE]: Term 1 election: Requested vote from peers e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201), 8c899ae4122d4a4fa4aa06b02a2ca2d3 (127.21.123.66:45807)
14:03:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.601379 22544 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4bd3f6916eba4f2fb1a9aace41281511" candidate_uuid: "0ed313e058b5467f883ba31f1d164b3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3"
14:03:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.601389 22403 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "4bd3f6916eba4f2fb1a9aace41281511" candidate_uuid: "0ed313e058b5467f883ba31f1d164b3d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e6afa1c1602247b682b32d093471e18d"
14:03:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.601545 22403 raft_consensus.cc:3058] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d [term 0 FOLLOWER]: Advancing to term 1
14:03:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.601575 22544 raft_consensus.cc:3058] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 0 FOLLOWER]: Advancing to term 1
14:03:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.603366 22403 raft_consensus.cc:2466] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0ed313e058b5467f883ba31f1d164b3d in term 1.
14:03:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.603433 22544 raft_consensus.cc:2466] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0ed313e058b5467f883ba31f1d164b3d in term 1.
14:03:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.603801 22610 leader_election.cc:304] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0ed313e058b5467f883ba31f1d164b3d, e6afa1c1602247b682b32d093471e18d; no voters:
14:03:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.604101 22764 raft_consensus.cc:2802] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [term 1 FOLLOWER]: Leader election won for term 1
14:03:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.604271 22764 raft_consensus.cc:695] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [term 1 LEADER]: Becoming Leader. State: Replica: 0ed313e058b5467f883ba31f1d164b3d, State: Running, Role: LEADER
14:03:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.604557 22764 consensus_queue.cc:237] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } }
14:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.606005 22771 tablet_replica.cc:440] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: TxnStatusTablet state changed. Reason: New leader 0ed313e058b5467f883ba31f1d164b3d. Latest consensus state: current_term: 1 leader_uuid: "0ed313e058b5467f883ba31f1d164b3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } }
14:03:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.606263 22771 tablet_replica.cc:443] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: This TxnStatusTablet replica's current role is: LEADER
14:03:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.606824 22787 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
14:03:38.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.607142 22787 txn_status_manager.cc:930] Loading transaction status metadata into memory...
14:03:38.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.607632 22199 catalog_manager.cc:5582] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d reported cstate change: term changed from 0 to 1, leader changed from <none> to 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67). New cstate: current_term: 1 leader_uuid: "0ed313e058b5467f883ba31f1d164b3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } health_report { overall_health: HEALTHY } } }
14:03:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.648062 22403 raft_consensus.cc:1273] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d [term 1 FOLLOWER]: Refusing update from remote peer 0ed313e058b5467f883ba31f1d164b3d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:03:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.648113 22544 raft_consensus.cc:1273] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [term 1 FOLLOWER]: Refusing update from remote peer 0ed313e058b5467f883ba31f1d164b3d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:03:38.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.648988 22764 consensus_queue.cc:1035] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:03:38.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.649302 22771 consensus_queue.cc:1035] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:03:38.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.658413 22771 consensus_queue.cc:1035] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:03:38.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.661144 22767 tablet_replica.cc:440] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d: TxnStatusTablet state changed. Reason: New leader 0ed313e058b5467f883ba31f1d164b3d. Latest consensus state: current_term: 1 leader_uuid: "0ed313e058b5467f883ba31f1d164b3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } }
14:03:38.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.661456 22767 tablet_replica.cc:443] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d: This TxnStatusTablet replica's current role is: FOLLOWER
14:03:38.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.665210 22763 tablet_replica.cc:440] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: TxnStatusTablet state changed. Reason: New leader 0ed313e058b5467f883ba31f1d164b3d. Latest consensus state: current_term: 1 leader_uuid: "0ed313e058b5467f883ba31f1d164b3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } }
14:03:38.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.665474 22763 tablet_replica.cc:443] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: This TxnStatusTablet replica's current role is: FOLLOWER
14:03:38.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.670579 22790 mvcc.cc:204] Tried to move back new op lower bound from 7170815666772819968 to 7170815666606350336. Current Snapshot: MvccSnapshot[applied={T|T < 7170815666772819968}]
14:03:38.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.670744 22791 mvcc.cc:204] Tried to move back new op lower bound from 7170815666772819968 to 7170815666606350336. Current Snapshot: MvccSnapshot[applied={T|T < 7170815666772819968}]
14:03:38.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.674149 22771 tablet_replica.cc:440] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0ed313e058b5467f883ba31f1d164b3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } }
14:03:38.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.674412 22771 tablet_replica.cc:443] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: This TxnStatusTablet replica's current role is: LEADER
14:03:38.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.674690 22771 tablet_replica.cc:440] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0ed313e058b5467f883ba31f1d164b3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } }
14:03:38.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.674871 22771 tablet_replica.cc:443] T 4bd3f6916eba4f2fb1a9aace41281511 P 0ed313e058b5467f883ba31f1d164b3d: This TxnStatusTablet replica's current role is: LEADER
14:03:38.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.679028 22767 tablet_replica.cc:440] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0ed313e058b5467f883ba31f1d164b3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } }
14:03:38.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.679216 22767 tablet_replica.cc:443] T 4bd3f6916eba4f2fb1a9aace41281511 P e6afa1c1602247b682b32d093471e18d: This TxnStatusTablet replica's current role is: FOLLOWER
14:03:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.682524 22792 mvcc.cc:204] Tried to move back new op lower bound from 7170815666772819968 to 7170815666606350336. Current Snapshot: MvccSnapshot[applied={T|T < 7170815666772819968}]
14:03:38.685 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
14:03:38.692 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 08d3968b837044f989a5b012b478ceae
14:03:38.693 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.123.124:38761
14:03:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.693195 22763 tablet_replica.cc:440] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0ed313e058b5467f883ba31f1d164b3d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 } } peers { permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" member_type: VOTER last_known_addr { host: "127.21.123.66" port: 45807 } } peers { permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 } } }
14:03:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.693430 22763 tablet_replica.cc:443] T 4bd3f6916eba4f2fb1a9aace41281511 P 8c899ae4122d4a4fa4aa06b02a2ca2d3: This TxnStatusTablet replica's current role is: FOLLOWER
14:03:38.695 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:03:38.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.695866 21997 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu pid 22167
14:03:38.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.697342 22241 generic_service.cc:196] Checking for leaks (request via RPC)
14:03:38.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.703495 22771 consensus_queue.cc:1035] T a382527694d540cb9a48ab998871f9ff P 0ed313e058b5467f883ba31f1d164b3d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:03:38.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.760152 22763 consensus_queue.cc:1035] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e6afa1c1602247b682b32d093471e18d" member_type: VOTER last_known_addr { host: "127.21.123.65" port: 45201 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:03:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.773392 22773 consensus_queue.cc:1035] T 79e2cea363a6408d9a53d72158f80fa2 P 8c899ae4122d4a4fa4aa06b02a2ca2d3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" member_type: VOTER last_known_addr { host: "127.21.123.67" port: 44431 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:03:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:38.852344 21997 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu with pid 22167
14:03:38.873 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.123.124:38761(127.21.123.124:38761)] handling channelInactive
14:03:38.873 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state READY due to: connection closed
14:03:38.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:38.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:38.873158 22470 connection.cc:537] client connection to 127.21.123.124:38761 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:03:38.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:38.874454 22025 connection.cc:537] server connection from 127.0.0.1:46064 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:03:38.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:38.874650 22589 heartbeater.cc:646] Failed to heartbeat to 127.21.123.124:38761 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:03:38.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:38.875054 22448 heartbeater.cc:646] Failed to heartbeat to 127.21.123.124:38761 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.21.123.124:38761 (error 108)
14:03:38.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:38.875530 22104 connection.cc:537] server connection from 127.0.0.1:54212 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:03:38.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:38.875572 22731 heartbeater.cc:646] Failed to heartbeat to 127.21.123.124:38761 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.21.123.124:38761 (error 108)
14:03:39.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.152143 22805 raft_consensus.cc:491] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 289f4dff4a8f4197a18cdb96452c9887)
14:03:39.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.152344 22805 raft_consensus.cc:513] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.153026 22806 raft_consensus.cc:491] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 289f4dff4a8f4197a18cdb96452c9887)
14:03:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.153187 22806 raft_consensus.cc:513] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.153789 22805 leader_election.cc:290] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049), 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761)
14:03:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.154247 22048 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "891a087227fa4198b42224b537877725" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "c51d2fbfbf0a4b38819a021299252b73" is_pre_election: true
14:03:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.154469 22048 raft_consensus.cc:2466] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 891a087227fa4198b42224b537877725 in term 1.
14:03:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.155001 22128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c51d2fbfbf0a4b38819a021299252b73" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "891a087227fa4198b42224b537877725" is_pre_election: true
14:03:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.155213 22128 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c51d2fbfbf0a4b38819a021299252b73 in term 1.
14:03:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.155282 22806 leader_election.cc:290] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 891a087227fa4198b42224b537877725 (127.21.123.125:36393), 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761)
14:03:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.155457 22105 leader_election.cc:304] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 891a087227fa4198b42224b537877725, c51d2fbfbf0a4b38819a021299252b73; no voters:
14:03:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.156061 22805 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:03:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.156067 22026 leader_election.cc:304] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 891a087227fa4198b42224b537877725, c51d2fbfbf0a4b38819a021299252b73; no voters:
14:03:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.156176 22805 raft_consensus.cc:491] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 289f4dff4a8f4197a18cdb96452c9887)
14:03:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.156291 22805 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 1 FOLLOWER]: Advancing to term 2
14:03:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.156440 22806 raft_consensus.cc:2802] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:03:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.156541 22806 raft_consensus.cc:491] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 289f4dff4a8f4197a18cdb96452c9887)
14:03:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.156630 22806 raft_consensus.cc:3058] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 1 FOLLOWER]: Advancing to term 2
14:03:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:39.156901 22106 leader_election.cc:336] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761): Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:39.157614 22025 leader_election.cc:336] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761): Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.159395 22805 raft_consensus.cc:513] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.159708 22806 raft_consensus.cc:513] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:39.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.160076 22805 leader_election.cc:290] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 2 election: Requested vote from peers c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049), 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761)
14:03:39.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.160322 22806 leader_election.cc:290] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 2 election: Requested vote from peers 891a087227fa4198b42224b537877725 (127.21.123.125:36393), 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761)
14:03:39.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.160492 22048 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "891a087227fa4198b42224b537877725" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "c51d2fbfbf0a4b38819a021299252b73"
14:03:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.160758 22048 raft_consensus.cc:2391] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 891a087227fa4198b42224b537877725 in current term 2: Already voted for candidate c51d2fbfbf0a4b38819a021299252b73 in this term.
14:03:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.161204 22128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c51d2fbfbf0a4b38819a021299252b73" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "891a087227fa4198b42224b537877725"
14:03:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.161442 22128 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate c51d2fbfbf0a4b38819a021299252b73 in current term 2: Already voted for candidate 891a087227fa4198b42224b537877725 in this term.
14:03:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:39.161701 22025 leader_election.cc:336] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761): Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:39.161727 22106 leader_election.cc:336] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761): Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.161850 22106 leader_election.cc:304] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 891a087227fa4198b42224b537877725; no voters: 289f4dff4a8f4197a18cdb96452c9887, c51d2fbfbf0a4b38819a021299252b73
14:03:39.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.161959 22026 leader_election.cc:304] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c51d2fbfbf0a4b38819a021299252b73; no voters: 289f4dff4a8f4197a18cdb96452c9887, 891a087227fa4198b42224b537877725
14:03:39.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.162114 22805 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
14:03:39.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.162357 22806 raft_consensus.cc:2747] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
14:03:39.195 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.196 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.197 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.198 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.21.123.124:38761(127.21.123.124:38761) for tablet Kudu Master: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.215 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.216 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.216 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.216 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.217 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.235 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.236 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.236 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.236 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.237 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.255 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.256 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.256 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.256 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.257 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.275 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.276 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.276 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.276 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.277 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.315 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.316 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.316 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.316 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.317 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.355 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.356 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.356 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.356 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.357 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.435 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.436 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.436 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.436 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.437 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.438 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] sending RPC to server master-127.21.123.124:38761, [3ms] received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [3ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [121ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [241ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [242ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, deferred=Deferred@492179085(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@239c4e02 -> (continuation of Deferred@151348030 after retry RPC after error@1685632220) -> (continuation of Deferred@1232596152 after retry RPC after error@148522954) -> (continuation of Deferred@719640956 after retry RPC after error@186336936) -> (continuation of Deferred@2105502047 after retry RPC after error@666993965) -> (continuation of Deferred@727877520 after retry RPC after error@2048764498) -> (continuation of Deferred@2083389919 after retry RPC after error@1466764186), errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@680c48e0 -> (continuation of Deferred@151348030 after retry RPC after error@1685632220) -> (continuation of Deferred@1232596152 after retry RPC after error@148522954) -> (continuation of Deferred@719640956 after retry RPC after error@186336936) -> (continuation of Deferred@2105502047 after retry RPC after error@666993965) -> (continuation of Deferred@727877520 after retry RPC after error@2048764498) -> (continuation of Deferred@2083389919 after retry RPC after error@1466764186)))
14:03:39.439 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] sending RPC to server master-127.21.123.124:38761, [3ms] received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [3ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [121ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [241ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [242ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, deferred=Deferred@492179085(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@239c4e02 -> (continuation of Deferred@151348030 after retry RPC after error@1685632220) -> (continuation of Deferred@1232596152 after retry RPC after error@148522954) -> (continuation of Deferred@719640956 after retry RPC after error@186336936) -> (continuation of Deferred@2105502047 after retry RPC after error@666993965) -> (continuation of Deferred@727877520 after retry RPC after error@2048764498) -> (continuation of Deferred@2083389919 after retry RPC after error@1466764186), errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@680c48e0 -> (continuation of Deferred@151348030 after retry RPC after error@1685632220) -> (continuation of Deferred@1232596152 after retry RPC after error@148522954) -> (continuation of Deferred@719640956 after retry RPC after error@186336936) -> (continuation of Deferred@2105502047 after retry RPC after error@666993965) -> (continuation of Deferred@727877520 after retry RPC after error@2048764498) -> (continuation of Deferred@2083389919 after retry RPC after error@1466764186)))
14:03:39.455 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.456 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.456 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.456 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.457 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.475 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.476 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.476 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.476 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.477 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.495 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.496 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.496 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.496 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.496 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.515 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.516 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.516 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.516 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.517 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.535 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.536 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.536 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.536 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.536 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.573634 22806 raft_consensus.cc:491] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:39.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.573887 22806 raft_consensus.cc:513] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:39.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.574514 22806 leader_election.cc:290] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 891a087227fa4198b42224b537877725 (127.21.123.125:36393), 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761)
14:03:39.575 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.575095 22128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c51d2fbfbf0a4b38819a021299252b73" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "891a087227fa4198b42224b537877725" is_pre_election: true
14:03:39.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.575342 22128 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c51d2fbfbf0a4b38819a021299252b73 in term 2.
14:03:39.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.575841 22026 leader_election.cc:304] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 891a087227fa4198b42224b537877725, c51d2fbfbf0a4b38819a021299252b73; no voters:
14:03:39.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:39.576020 22025 leader_election.cc:336] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761): Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:39.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.576165 22806 raft_consensus.cc:2802] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 2 FOLLOWER]: Leader pre-election won for term 3
14:03:39.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.576268 22806 raft_consensus.cc:491] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:03:39.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.576351 22806 raft_consensus.cc:3058] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 2 FOLLOWER]: Advancing to term 3
14:03:39.576 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.577 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.577 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.578387 22806 raft_consensus.cc:513] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:39.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.578907 22806 leader_election.cc:290] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 3 election: Requested vote from peers 891a087227fa4198b42224b537877725 (127.21.123.125:36393), 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761)
14:03:39.579 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:39.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.579355 22128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c51d2fbfbf0a4b38819a021299252b73" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "891a087227fa4198b42224b537877725"
14:03:39.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.579535 22128 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 2 FOLLOWER]: Advancing to term 3
14:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:39.580246 22025 leader_election.cc:336] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761): Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111)
14:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.581629 22128 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c51d2fbfbf0a4b38819a021299252b73 in term 3.
14:03:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.582010 22026 leader_election.cc:304] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 891a087227fa4198b42224b537877725, c51d2fbfbf0a4b38819a021299252b73; no voters: 289f4dff4a8f4197a18cdb96452c9887
14:03:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.582346 22806 raft_consensus.cc:2802] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 3 FOLLOWER]: Leader election won for term 3
14:03:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.582803 22806 raft_consensus.cc:695] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 3 LEADER]: Becoming Leader. State: Replica: c51d2fbfbf0a4b38819a021299252b73, State: Running, Role: LEADER
14:03:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.583261 22806 consensus_queue.cc:237] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:39.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.584754 22815 sys_catalog.cc:455] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c51d2fbfbf0a4b38819a021299252b73. Latest consensus state: current_term: 3 leader_uuid: "c51d2fbfbf0a4b38819a021299252b73" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:39.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.584986 22815 sys_catalog.cc:458] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: This master's current role is: LEADER
14:03:39.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.585543 22816 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:03:39.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.587852 22816 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae]
14:03:39.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.588340 22816 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=71e4144517084e3087907b27d7a03a66]
14:03:39.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.589978 22816 tablet_loader.cc:96] loaded metadata for tablet 4bd3f6916eba4f2fb1a9aace41281511 (table kudu_system.kudu_transactions [id=71e4144517084e3087907b27d7a03a66])
14:03:39.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.590457 22816 tablet_loader.cc:96] loaded metadata for tablet 79e2cea363a6408d9a53d72158f80fa2 (table txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae])
14:03:39.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.590816 22816 tablet_loader.cc:96] loaded metadata for tablet a382527694d540cb9a48ab998871f9ff (table txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae])
14:03:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.591102 22816 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:03:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.591943 22816 catalog_manager.cc:1261] Loaded cluster ID: e33ca72474a94b3695af9e40ba7dd46c
14:03:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.592048 22816 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:03:39.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.593345 22816 catalog_manager.cc:1506] Loading token signing keys...
14:03:39.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.594198 22816 catalog_manager.cc:5966] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: Loaded TSK: 0
14:03:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.595038 22816 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:03:39.635 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:39.636 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.636 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.636 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.124:38761: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761
14:03:39.639 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:03:39.640 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=184), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [1ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [1ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [119ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [181ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: Network error: Failed to connect to peer master-127.21.123.124:38761(127.21.123.124:38761): Connection refused: /127.21.123.124:38761, [181ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, deferred=Deferred@1356731855(state=PENDING, result=null, callback=(continuation of Deferred@785746681 after retry RPC after error@2080017713) -> (continuation of Deferred@1784127911 after retry RPC after error@1250402639) -> (continuation of Deferred@1208445092 after retry RPC after error@1740371856) -> (continuation of Deferred@817248345 after retry RPC after error@1196190425) -> (continuation of Deferred@880634076 after retry RPC after error@214048436) -> (continuation of Deferred@221744005 after retry RPC after error@688468568), errback=(continuation of Deferred@785746681 after retry RPC after error@2080017713) -> (continuation of Deferred@1784127911 after retry RPC after error@1250402639) -> (continuation of Deferred@1208445092 after retry RPC after error@1740371856) -> (continuation of Deferred@817248345 after retry RPC after error@1196190425) -> (continuation of Deferred@880634076 after retry RPC after error@214048436) -> (continuation of Deferred@221744005 after retry RPC after error@688468568))) after lookup
14:03:39.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:39.669274 22025 consensus_peers.cc:487] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 -> Peer 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761): Couldn't send request to peer 289f4dff4a8f4197a18cdb96452c9887. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.690708 22038 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47462:
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:03:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:03:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:39.692571 22038 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:03:39.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.706748 22128 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 3 FOLLOWER]: Refusing update from remote peer c51d2fbfbf0a4b38819a021299252b73: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
14:03:39.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.707312 22815 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [LEADER]: Connected to new peer: Peer: permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
14:03:39.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:39.708698 22818 meta_cache.cc:302] tablet 4bd3f6916eba4f2fb1a9aace41281511: replica 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67:44431) has failed: Timed out: connection negotiation to 127.21.123.67:44431 for RPC CoordinateTransaction timed out after 0.005s (ON_OUTBOUND_QUEUE)
14:03:39.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:39.710978 22069 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:47444 (request call id 15) took 69 ms (client timeout 65 ms). Trace:
14:03:39.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:39.711174 22069 rpcz_store.cc:269] 0623 14:03:39.641507 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:03:39.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:39.641568 (+ 61us) service_pool.cc:225] Handling call
14:03:39.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:39.710912 (+ 69344us) inbound_call.cc:173] Queueing success response
14:03:39.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":702,"init.queue_time_us":758,"init.run_cpu_time_us":9463,"init.run_wall_time_us":54572,"thread_start_us":1553,"threads_started":5}
14:03:39.713 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: Failed to write to server: (no server available): CoordinateTransaction(txn_status_tablet_id: "4bd3f6916eba4f2fb1a9aace41281511" op { type: KEEP_TXN_ALIVE txn_id: 0 user: "slave" }) passed its deadline: Timed out: connection negotiation to 127.21.123.67:44431 for RPC CoordinateTransaction timed out after 0.005s (ON_OUTBOUND_QUEUE)
14:03:39.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.715492 22829 sys_catalog.cc:455] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c51d2fbfbf0a4b38819a021299252b73. Latest consensus state: current_term: 3 leader_uuid: "c51d2fbfbf0a4b38819a021299252b73" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:39.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.715735 22829 sys_catalog.cc:458] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: This master's current role is: FOLLOWER
14:03:39.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.717257 22815 sys_catalog.cc:455] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "c51d2fbfbf0a4b38819a021299252b73" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:39.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.717478 22815 sys_catalog.cc:458] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: This master's current role is: LEADER
14:03:39.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.717957 22829 sys_catalog.cc:455] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "c51d2fbfbf0a4b38819a021299252b73" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:39.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:39.718137 22829 sys_catalog.cc:458] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: This master's current role is: FOLLOWER
14:03:40.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:40.151504 22025 consensus_peers.cc:487] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 -> Peer 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761): Couldn't send request to peer 289f4dff4a8f4197a18cdb96452c9887. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:03:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:40.366906 22025 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111) [suppressed 18 similar messages]
14:03:40.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:40.643715 22025 consensus_peers.cc:487] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 -> Peer 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761): Couldn't send request to peer 289f4dff4a8f4197a18cdb96452c9887. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
14:03:40.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:40.778692 22449 heartbeater.cc:499] Master 127.21.123.126:34049 was elected leader, sending a full tablet report...
14:03:40.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a382527694d540cb9a48ab998871f9ff for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
14:03:40.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 79e2cea363a6408d9a53d72158f80fa2 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
14:03:40.880 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=a382527694d540cb9a48ab998871f9ff@[0ed313e058b5467f883ba31f1d164b3d(127.21.123.67:44431)[L],8c899ae4122d4a4fa4aa06b02a2ca2d3(127.21.123.66:45807),e6afa1c1602247b682b32d093471e18d(127.21.123.65:45201)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=79e2cea363a6408d9a53d72158f80fa2@[0ed313e058b5467f883ba31f1d164b3d(127.21.123.67:44431),8c899ae4122d4a4fa4aa06b02a2ca2d3(127.21.123.66:45807)[L],e6afa1c1602247b682b32d093471e18d(127.21.123.65:45201)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
14:03:40.881 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.123.126:34049, [2ms] Sub RPC GetTableLocations: received response from server master-127.21.123.126:34049: OK, deferred=null) after lookup
14:03:40.881 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0ed313e058b5467f883ba31f1d164b3d(127.21.123.67:44431)] connecting to peer
14:03:40.882 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 0ed313e058b5467f883ba31f1d164b3d(127.21.123.67:44431)] Successfully connected to peer
14:03:40.893 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa5989352, L:/127.0.0.1:59118 - R:/127.21.123.67:44431] using TOKEN/null
14:03:40.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:40.897588 22637 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.21.123.124:38761)
14:03:40.930 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.21.123.124:38761
14:03:40.930 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:40.931123 21997 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/logs
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data/info.pb
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.124:38761
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.124
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45117
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41795/jwks.json
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:03:40.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:03:41.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.153506 22025 consensus_peers.cc:487] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 -> Peer 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761): Couldn't send request to peer 289f4dff4a8f4197a18cdb96452c9887. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
14:03:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.202622 22732 heartbeater.cc:499] Master 127.21.123.126:34049 was elected leader, sending a full tablet report...
14:03:41.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.254902 22855 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:03:41.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.255177 22855 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:41.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.255247 22855 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:03:41.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.262948 22855 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:03:41.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.263031 22855 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:03:41.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.263095 22855 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:03:41.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.263134 22855 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:03:41.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.263186 22855 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:03:41.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.263223 22855 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:03:41.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.263265 22855 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41795/jwks.json
14:03:41.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.272603 22855 master_runner.cc:386] Master server non-default flags:
14:03:41.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:41.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:41.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:41.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:41.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:03:41.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.124:38761
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41795/jwks.json
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data/info.pb
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.124
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45117
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/logs
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6732
14:03:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:03:41.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.274392 22855 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:41.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.276218 22855 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:03:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.286407 22861 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.286471 22864 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.286444 22862 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:41.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.287357 22855 server_base.cc:1048] running on GCE node
14:03:41.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.288087 22855 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:41.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.289516 22855 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:41.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.290737 22855 hybrid_clock.cc:648] HybridClock initialized: now 1750687421290659 us; error 50 us; skew 500 ppm
14:03:41.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.291107 22855 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:41.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.299857 22008 master_runner.cc:186] Error getting master registration for 127.21.123.124:38761: Network error: Client connection negotiation failed: client connection to 127.21.123.124:38761: connect: Connection refused (error 111),
14:03:41.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.300063 22008 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:03:41.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.300779 22855 webserver.cc:469] Webserver started at http://127.21.123.124:45117/ using document root <none> and password file <none>
14:03:41.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.301476 22855 fs_manager.cc:362] Metadata directory not provided
14:03:41.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.301591 22855 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:41.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.306841 22855 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.003s sys 0.001s
14:03:41.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.309345 22870 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.310591 22855 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.310719 22855 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data,/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "289f4dff4a8f4197a18cdb96452c9887"
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:36 on dist-test-slave-stbh"
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.311376 22855 fs_report.cc:389] FS layout report
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/wal
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data/data
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:41.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.327947 22855 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:41.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.328722 22855 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:41.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.329005 22855 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.351229 22855 rpc_server.cc:307] RPC server started. Bound to: 127.21.123.124:38761
14:03:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.351261 22931 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.123.124:38761 every 8 connection(s)
14:03:41.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.352838 22855 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-2/data/info.pb
14:03:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.357096 21997 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu as pid 22855
14:03:41.357 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:41.357 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.123.126:34049
14:03:41.357 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:03:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.357853 21997 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu pid 22008
14:03:41.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.358935 22079 generic_service.cc:196] Checking for leaks (request via RPC)
14:03:41.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.360009 22932 sys_catalog.cc:263] Verifying existing consensus state
14:03:41.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.365180 22932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: Bootstrap starting.
14:03:41.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.387910 22932 log.cc:826] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: Log is configured to *not* fsync() on all Append() calls
14:03:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.391953 22590 heartbeater.cc:499] Master 127.21.123.126:34049 was elected leader, sending a full tablet report...
14:03:41.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.404275 22589 heartbeater.cc:344] Connected to a master server at 127.21.123.124:38761
14:03:41.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.404603 22932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:03:41.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.405205 22932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: Bootstrap complete.
14:03:41.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.413491 22932 raft_consensus.cc:357] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:41.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.414412 22932 raft_consensus.cc:738] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 289f4dff4a8f4197a18cdb96452c9887, State: Initialized, Role: FOLLOWER
14:03:41.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.415103 22932 consensus_queue.cc:260] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:41.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.415823 22938 sys_catalog.cc:455] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:41.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.416244 22938 sys_catalog.cc:458] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: This master's current role is: FOLLOWER
14:03:41.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.417028 22932 sys_catalog.cc:564] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: configured and running, proceeding with master startup.
14:03:41.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.426762 22949 catalog_manager.cc:1261] Loaded cluster ID: e33ca72474a94b3695af9e40ba7dd46c
14:03:41.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.426973 22949 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: loading cluster ID for follower catalog manager: success
14:03:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.431319 22949 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: acquiring CA information for follower catalog manager: success
14:03:41.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.433874 22949 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:03:41.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.508915 21997 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu with pid 22008
14:03:41.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.530017 22469 connection.cc:537] client connection to 127.21.123.126:34049 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:03:41.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.530166 22105 connection.cc:537] server connection from 127.0.0.1:54194 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:03:41.531 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:41.531 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.123.126:34049(127.21.123.126:34049)] handling channelInactive
14:03:41.531 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.126:34049(127.21.123.126:34049)] cleaning up while in state READY due to: connection closed
14:03:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.532326 22951 negotiation.cc:337] Failed RPC negotiation. Trace:
14:03:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:41.423675 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.21.123.126:34049 (local address 127.0.0.1:47470)
14:03:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:41.438304 (+ 14629us) negotiation.cc:107] Waiting for socket to connect
14:03:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:41.438342 (+ 38us) client_negotiation.cc:174] Beginning negotiation
14:03:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:41.438514 (+ 172us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
14:03:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:41.530887 (+ 92373us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:03:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":14400,"thread_start_us":14248,"threads_started":1}
14:03:41.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.532737 22855 master_runner.cc:186] Error getting master registration for 127.21.123.126:34049: Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
14:03:41.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.538419 22855 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:03:41.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.669967 22938 raft_consensus.cc:491] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:41.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.670300 22938 raft_consensus.cc:513] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.672395 22938 leader_election.cc:290] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049), 891a087227fa4198b42224b537877725 (127.21.123.125:36393)
14:03:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.673260 22872 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: connect: Connection refused (error 111)
14:03:41.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.683597 22872 leader_election.cc:336] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049): Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: connect: Connection refused (error 111)
14:03:41.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.690124 22128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "289f4dff4a8f4197a18cdb96452c9887" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "891a087227fa4198b42224b537877725" is_pre_election: true
14:03:41.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.691288 22874 leader_election.cc:304] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 289f4dff4a8f4197a18cdb96452c9887; no voters: 891a087227fa4198b42224b537877725, c51d2fbfbf0a4b38819a021299252b73
14:03:41.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.691792 22938 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:03:41.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.810582 22957 raft_consensus.cc:491] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader c51d2fbfbf0a4b38819a021299252b73)
14:03:41.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.810757 22957 raft_consensus.cc:513] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:41.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.811370 22957 leader_election.cc:290] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049), 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761)
14:03:41.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.812450 22105 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: connect: Connection refused (error 111) [suppressed 6 similar messages]
14:03:41.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.814050 22105 leader_election.cc:336] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049): Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: connect: Connection refused (error 111)
14:03:41.815 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.126:34049(127.21.123.126:34049)] connecting to peer
14:03:41.816 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049
14:03:41.816 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.126:34049(127.21.123.126:34049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049
14:03:41.816 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.21.123.126:34049(127.21.123.126:34049) for tablet Kudu Master: Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049
14:03:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.818480 22897 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "891a087227fa4198b42224b537877725" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "289f4dff4a8f4197a18cdb96452c9887" is_pre_election: true
14:03:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.818871 22897 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 891a087227fa4198b42224b537877725 in term 1.
14:03:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.819401 22106 leader_election.cc:304] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 289f4dff4a8f4197a18cdb96452c9887, 891a087227fa4198b42224b537877725; no voters: c51d2fbfbf0a4b38819a021299252b73
14:03:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.819697 22957 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 3 FOLLOWER]: Leader pre-election won for term 4
14:03:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.819814 22957 raft_consensus.cc:491] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 3 FOLLOWER]: Starting leader election (detected failure of leader c51d2fbfbf0a4b38819a021299252b73)
14:03:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.819923 22957 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 3 FOLLOWER]: Advancing to term 4
14:03:41.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.821915 22957 raft_consensus.cc:513] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:41.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.822499 22957 leader_election.cc:290] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 4 election: Requested vote from peers c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049), 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761)
14:03:41.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.822999 22897 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "891a087227fa4198b42224b537877725" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "289f4dff4a8f4197a18cdb96452c9887"
14:03:41.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.823204 22897 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 1 FOLLOWER]: Advancing to term 4
14:03:41.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.823709 22105 leader_election.cc:336] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049): Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: connect: Connection refused (error 111)
14:03:41.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.826823 22897 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 891a087227fa4198b42224b537877725 in term 4.
14:03:41.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.827180 22106 leader_election.cc:304] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 289f4dff4a8f4197a18cdb96452c9887, 891a087227fa4198b42224b537877725; no voters: c51d2fbfbf0a4b38819a021299252b73
14:03:41.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.827431 22957 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 4 FOLLOWER]: Leader election won for term 4
14:03:41.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.827841 22957 raft_consensus.cc:695] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [term 4 LEADER]: Becoming Leader. State: Replica: 891a087227fa4198b42224b537877725, State: Running, Role: LEADER
14:03:41.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.828276 22957 consensus_queue.cc:237] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:41.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.829788 22961 sys_catalog.cc:455] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 891a087227fa4198b42224b537877725. Latest consensus state: current_term: 4 leader_uuid: "891a087227fa4198b42224b537877725" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:41.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.830036 22961 sys_catalog.cc:458] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: This master's current role is: LEADER
14:03:41.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.830585 22963 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:03:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.833009 22963 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae]
14:03:41.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.833503 22963 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=71e4144517084e3087907b27d7a03a66]
14:03:41.835 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.124:38761(127.21.123.124:38761)] connecting to peer
14:03:41.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.835007 22963 tablet_loader.cc:96] loaded metadata for tablet 4bd3f6916eba4f2fb1a9aace41281511 (table kudu_system.kudu_transactions [id=71e4144517084e3087907b27d7a03a66])
14:03:41.835 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.126:34049(127.21.123.126:34049)] connecting to peer
14:03:41.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.835523 22963 tablet_loader.cc:96] loaded metadata for tablet 79e2cea363a6408d9a53d72158f80fa2 (table txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae])
14:03:41.835 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.123.124:38761(127.21.123.124:38761)] Successfully connected to peer
14:03:41.836 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049
14:03:41.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.836040 22963 tablet_loader.cc:96] loaded metadata for tablet a382527694d540cb9a48ab998871f9ff (table txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae])
14:03:41.836 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.126:34049(127.21.123.126:34049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049
14:03:41.836 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.126:34049: Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049
14:03:41.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.836436 22963 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:03:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.837580 22963 catalog_manager.cc:1261] Loaded cluster ID: e33ca72474a94b3695af9e40ba7dd46c
14:03:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.837684 22963 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:03:41.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.839133 22963 catalog_manager.cc:1506] Loading token signing keys...
14:03:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.840229 22963 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725: Loaded TSK: 0
14:03:41.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.841284 22963 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:03:41.846 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x6a4baa7f, L:/127.0.0.1:40978 - R:/127.21.123.124:38761] using TOKEN/null
14:03:41.848 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049; will retry
14:03:41.855 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.126:34049(127.21.123.126:34049)] connecting to peer
14:03:41.855 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049
14:03:41.856 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.126:34049(127.21.123.126:34049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049
14:03:41.856 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.126:34049: Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049
14:03:41.857 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:03:41.858 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=3, TimeoutTracker(timeout=250, elapsed=42), Traces: [0ms] sending RPC to server master-127.21.123.126:34049, [1ms] received response from server master-127.21.123.126:34049: Network error: Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: Network error: Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, [33ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: OK, [33ms] delaying RPC due to: Service unavailable: Master config (127.21.123.125:36393,127.21.123.124:38761,127.21.123.126:34049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: Network error: Failed to connect to peer master-127.21.123.126:34049(127.21.123.126:34049): Connection refused: /127.21.123.126:34049, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: OK, deferred=Deferred@2037251638(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@1858b9ba -> (continuation of Deferred@1205031166 after retry RPC after error@37961046), errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@1449ef66 -> (continuation of Deferred@1205031166 after retry RPC after error@37961046))) after lookup
14:03:41.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.887667 22448 heartbeater.cc:344] Connected to a master server at 127.21.123.124:38761
14:03:41.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.890110 22887 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "e6afa1c1602247b682b32d093471e18d" instance_seqno: 1750687416991477) as {username='slave'} at 127.21.123.65:51909; Asking this server to re-register.
14:03:41.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.890797 22448 heartbeater.cc:461] Registering TS with master...
14:03:41.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.892612 22887 ts_manager.cc:194] Registered new tserver with Master: e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201)
14:03:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.898135 22119 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45310:
14:03:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:03:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:03:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:03:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:03:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:41.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:03:41.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.903158 22119 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:03:41.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:41.935814 22105 consensus_peers.cc:487] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 -> Peer c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049): Couldn't send request to peer c51d2fbfbf0a4b38819a021299252b73. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:03:41.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.942399 22897 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 4 FOLLOWER]: Refusing update from remote peer 891a087227fa4198b42224b537877725: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
14:03:41.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.943168 22961 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [LEADER]: Connected to new peer: Peer: permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
14:03:41.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.946336 22976 sys_catalog.cc:455] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "891a087227fa4198b42224b537877725" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:41.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.946631 22976 sys_catalog.cc:458] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: This master's current role is: FOLLOWER
14:03:41.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.947219 22938 sys_catalog.cc:455] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 891a087227fa4198b42224b537877725. Latest consensus state: current_term: 4 leader_uuid: "891a087227fa4198b42224b537877725" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:41.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.947438 22938 sys_catalog.cc:458] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: This master's current role is: FOLLOWER
14:03:41.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.947686 22957 sys_catalog.cc:455] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "891a087227fa4198b42224b537877725" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:41.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.947858 22938 sys_catalog.cc:455] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "891a087227fa4198b42224b537877725" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:41.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.947968 22957 sys_catalog.cc:458] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 [sys.catalog]: This master's current role is: LEADER
14:03:41.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:41.948022 22938 sys_catalog.cc:458] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: This master's current role is: FOLLOWER
14:03:42.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:42.209194 22731 heartbeater.cc:344] Connected to a master server at 127.21.123.124:38761
14:03:42.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:42.210533 22887 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" instance_seqno: 1750687417976505) as {username='slave'} at 127.21.123.67:46631; Asking this server to re-register.
14:03:42.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:42.211100 22731 heartbeater.cc:461] Registering TS with master...
14:03:42.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:42.212158 22887 ts_manager.cc:194] Registered new tserver with Master: 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67:44431)
14:03:42.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:42.408710 22887 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" instance_seqno: 1750687417461194) as {username='slave'} at 127.21.123.66:35395; Asking this server to re-register.
14:03:42.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:42.409405 22589 heartbeater.cc:461] Registering TS with master...
14:03:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:42.410607 22887 ts_manager.cc:194] Registered new tserver with Master: 8c899ae4122d4a4fa4aa06b02a2ca2d3 (127.21.123.66:45807)
14:03:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:42.428880 22105 consensus_peers.cc:487] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 -> Peer c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049): Couldn't send request to peer c51d2fbfbf0a4b38819a021299252b73. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:03:42.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:42.530884 22591 heartbeater.cc:499] Master 127.21.123.125:36393 was elected leader, sending a full tablet report...
14:03:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:42.954520 22105 consensus_peers.cc:487] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 -> Peer c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049): Couldn't send request to peer c51d2fbfbf0a4b38819a021299252b73. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
14:03:43.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.206920 22733 heartbeater.cc:499] Master 127.21.123.125:36393 was elected leader, sending a full tablet report...
14:03:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.419503 22105 consensus_peers.cc:487] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 -> Peer c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049): Couldn't send request to peer c51d2fbfbf0a4b38819a021299252b73. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
14:03:43.535 [DEBUG - Time-limited test] (Connection.java:748) [peer 8c899ae4122d4a4fa4aa06b02a2ca2d3(127.21.123.66:45807)] connecting to peer
14:03:43.536 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 8c899ae4122d4a4fa4aa06b02a2ca2d3(127.21.123.66:45807)] Successfully connected to peer
14:03:43.548 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x90620824, L:/127.0.0.1:47166 - R:/127.21.123.66:45807] using TOKEN/null
14:03:43.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.557029 22508 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.21.123.124:38761)
14:03:43.588 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.21.123.126:34049
14:03:43.588 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.589056 21997 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/logs
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data/info.pb
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.126:34049
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.126
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38147
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41795/jwks.json
14:03:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:03:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:03:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.784214 22450 heartbeater.cc:499] Master 127.21.123.125:36393 was elected leader, sending a full tablet report...
14:03:43.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.910085 23005 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:03:43.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.910379 23005 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:43.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.910455 23005 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:03:43.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.918273 23005 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:03:43.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.918366 23005 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:03:43.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.918414 23005 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:03:43.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.918470 23005 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:03:43.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.918531 23005 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:03:43.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.918570 23005 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:03:43.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.918612 23005 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41795/jwks.json
14:03:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.923300 22105 consensus_peers.cc:487] T 00000000000000000000000000000000 P 891a087227fa4198b42224b537877725 -> Peer c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049): Couldn't send request to peer c51d2fbfbf0a4b38819a021299252b73. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.126:34049: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.928364 23005 master_runner.cc:386] Master server non-default flags:
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.123.84:33033
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.123.126:34049,127.21.123.125:36393,127.21.123.124:38761
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.123.126:34049
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41795/jwks.json
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data/info.pb
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.123.126
14:03:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38147
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/logs
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6732
14:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:03:43.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.930065 23005 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.931874 23005 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:03:43.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.941959 23011 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:43.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.942732 23014 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:43.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:43.943271 23012 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:43.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.944298 23005 server_base.cc:1048] running on GCE node
14:03:43.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.945099 23005 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:43.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.946528 23005 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:43.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.947746 23005 hybrid_clock.cc:648] HybridClock initialized: now 1750687423947683 us; error 42 us; skew 500 ppm
14:03:43.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.948050 23005 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:43.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.950357 23005 webserver.cc:469] Webserver started at http://127.21.123.126:38147/ using document root <none> and password file <none>
14:03:43.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.951083 23005 fs_manager.cc:362] Metadata directory not provided
14:03:43.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.951205 23005 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:43.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.956769 23005 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.002s sys 0.003s
14:03:43.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.959713 23020 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.961076 23005 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.961242 23005 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data,/tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c51d2fbfbf0a4b38819a021299252b73"
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-23 14:03:35 on dist-test-slave-stbh"
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.962055 23005 fs_report.cc:389] FS layout report
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/wal
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data/data
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:43.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.975492 23005 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:43.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.976384 23005 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:43.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:43.976655 23005 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:44.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.003893 23005 rpc_server.cc:307] RPC server started. Bound to: 127.21.123.126:34049
14:03:44.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.004190 23081 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.123.126:34049 every 8 connection(s)
14:03:44.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.006086 23005 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLQ_Kwd/test-tmp/mini-kudu-cluster7876230458048591282/master-0/data/info.pb
14:03:44.015 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:44.015 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.123.125:36393
14:03:44.016 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:03:44.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.015213 21997 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu as pid 23005
14:03:44.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.016229 21997 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu pid 22086
14:03:44.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.017479 22160 generic_service.cc:196] Checking for leaks (request via RPC)
14:03:44.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.030112 23083 sys_catalog.cc:263] Verifying existing consensus state
14:03:44.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.033598 23083 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: Bootstrap starting.
14:03:44.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.049793 23083 log.cc:826] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: Log is configured to *not* fsync() on all Append() calls
14:03:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.062960 23083 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:03:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.063565 23083 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: Bootstrap complete.
14:03:44.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.069566 23083 raft_consensus.cc:357] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:44.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.070379 23083 raft_consensus.cc:738] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: c51d2fbfbf0a4b38819a021299252b73, State: Initialized, Role: FOLLOWER
14:03:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.071126 23083 consensus_queue.cc:260] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:44.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.071960 23087 sys_catalog.cc:455] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:44.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.072270 23087 sys_catalog.cc:458] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: This master's current role is: FOLLOWER
14:03:44.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.072942 23083 sys_catalog.cc:564] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: configured and running, proceeding with master startup.
14:03:44.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.096161 23098 catalog_manager.cc:1261] Loaded cluster ID: e33ca72474a94b3695af9e40ba7dd46c
14:03:44.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.096303 23098 catalog_manager.cc:1554] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: loading cluster ID for follower catalog manager: success
14:03:44.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.100512 23098 catalog_manager.cc:1576] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: acquiring CA information for follower catalog manager: success
14:03:44.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.103319 23098 catalog_manager.cc:1604] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:03:44.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.171753 23047 raft_consensus.cc:3058] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 3 FOLLOWER]: Advancing to term 4
14:03:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.174297 21997 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu with pid 22086
14:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.189342 23104 sys_catalog.cc:455] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "891a087227fa4198b42224b537877725" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.189783 23104 sys_catalog.cc:458] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: This master's current role is: FOLLOWER
14:03:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.190274 23087 sys_catalog.cc:455] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 891a087227fa4198b42224b537877725. Latest consensus state: current_term: 4 leader_uuid: "891a087227fa4198b42224b537877725" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.190498 23087 sys_catalog.cc:458] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: This master's current role is: FOLLOWER
14:03:44.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:44.203982 23024 connection.cc:537] server connection from 127.0.0.1:47956 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:03:44.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:44.205323 23100 negotiation.cc:337] Failed RPC negotiation. Trace:
14:03:44.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:44.103288 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.21.123.125:36393 (local address 127.0.0.1:45322)
14:03:44.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:44.103543 (+ 255us) negotiation.cc:107] Waiting for socket to connect
14:03:44.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:44.103560 (+ 17us) client_negotiation.cc:174] Beginning negotiation
14:03:44.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:44.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:44.103659 (+ 99us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
14:03:44.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:44.204055 (+100396us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:03:44.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":141}
14:03:44.205 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.123.125:36393(127.21.123.125:36393)] handling channelInactive
14:03:44.207 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.125:36393(127.21.123.125:36393)] cleaning up while in state READY due to: connection closed
14:03:44.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.206034 23005 master_runner.cc:186] Error getting master registration for 127.21.123.125:36393: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
14:03:44.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.214514 23005 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:03:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.220738 22732 heartbeater.cc:344] Connected to a master server at 127.21.123.126:34049
14:03:44.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.223450 23037 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "0ed313e058b5467f883ba31f1d164b3d" instance_seqno: 1750687417976505) as {username='slave'} at 127.21.123.67:36047; Asking this server to re-register.
14:03:44.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.224133 22732 heartbeater.cc:461] Registering TS with master...
14:03:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.225575 23037 ts_manager.cc:194] Registered new tserver with Master: 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67:44431)
14:03:44.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.471753 23107 raft_consensus.cc:491] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 891a087227fa4198b42224b537877725)
14:03:44.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.472008 23107 raft_consensus.cc:513] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:44.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.472786 23107 leader_election.cc:290] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers c51d2fbfbf0a4b38819a021299252b73 (127.21.123.126:34049), 891a087227fa4198b42224b537877725 (127.21.123.125:36393)
14:03:44.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:44.476362 22874 leader_election.cc:336] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 891a087227fa4198b42224b537877725 (127.21.123.125:36393): Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111)
14:03:44.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.478683 23087 raft_consensus.cc:491] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 891a087227fa4198b42224b537877725)
14:03:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.478957 23087 raft_consensus.cc:513] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.480947 23087 leader_election.cc:290] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 891a087227fa4198b42224b537877725 (127.21.123.125:36393), 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761)
14:03:44.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:44.481491 23024 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111)
14:03:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.490098 22897 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c51d2fbfbf0a4b38819a021299252b73" candidate_term: 5 candidate_status { last_received { term: 4 index: 13 } } ignore_live_leader: false dest_uuid: "289f4dff4a8f4197a18cdb96452c9887" is_pre_election: true
14:03:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.490355 22897 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c51d2fbfbf0a4b38819a021299252b73 in term 4.
14:03:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.491021 23023 leader_election.cc:304] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 289f4dff4a8f4197a18cdb96452c9887, c51d2fbfbf0a4b38819a021299252b73; no voters:
14:03:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.491433 23087 raft_consensus.cc:2802] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 4 FOLLOWER]: Leader pre-election won for term 5
14:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.491547 23087 raft_consensus.cc:491] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 891a087227fa4198b42224b537877725)
14:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.491653 23087 raft_consensus.cc:3058] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 4 FOLLOWER]: Advancing to term 5
14:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.494927 23087 raft_consensus.cc:513] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.496809 23087 leader_election.cc:290] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 5 election: Requested vote from peers 891a087227fa4198b42224b537877725 (127.21.123.125:36393), 289f4dff4a8f4197a18cdb96452c9887 (127.21.123.124:38761)
14:03:44.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.496914 22897 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c51d2fbfbf0a4b38819a021299252b73" candidate_term: 5 candidate_status { last_received { term: 4 index: 13 } } ignore_live_leader: false dest_uuid: "289f4dff4a8f4197a18cdb96452c9887"
14:03:44.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.497130 22897 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 4 FOLLOWER]: Advancing to term 5
14:03:44.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.500082 22897 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c51d2fbfbf0a4b38819a021299252b73 in term 5.
14:03:44.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.500602 23023 leader_election.cc:304] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 289f4dff4a8f4197a18cdb96452c9887, c51d2fbfbf0a4b38819a021299252b73; no voters:
14:03:44.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.500924 23087 raft_consensus.cc:2802] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 5 FOLLOWER]: Leader election won for term 5
14:03:44.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.501158 23087 raft_consensus.cc:695] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [term 5 LEADER]: Becoming Leader. State: Replica: c51d2fbfbf0a4b38819a021299252b73, State: Running, Role: LEADER
14:03:44.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:44.501802 23024 leader_election.cc:336] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 891a087227fa4198b42224b537877725 (127.21.123.125:36393): Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111)
14:03:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.501705 23087 consensus_queue.cc:237] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 4.13, Last appended by leader: 13, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } }
14:03:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.504027 23104 sys_catalog.cc:455] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c51d2fbfbf0a4b38819a021299252b73. Latest consensus state: current_term: 5 leader_uuid: "c51d2fbfbf0a4b38819a021299252b73" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.504254 23104 sys_catalog.cc:458] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: This master's current role is: LEADER
14:03:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.505136 23113 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:03:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.505033 23047 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "289f4dff4a8f4197a18cdb96452c9887" candidate_term: 5 candidate_status { last_received { term: 4 index: 13 } } ignore_live_leader: false dest_uuid: "c51d2fbfbf0a4b38819a021299252b73" is_pre_election: true
14:03:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:44.505513 23024 leader_election.cc:336] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 891a087227fa4198b42224b537877725 (127.21.123.125:36393): Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111)
14:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.506227 22872 leader_election.cc:304] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 289f4dff4a8f4197a18cdb96452c9887; no voters: 891a087227fa4198b42224b537877725, c51d2fbfbf0a4b38819a021299252b73
14:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.506603 23107 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 5 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
14:03:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.508744 23113 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae]
14:03:44.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.509270 23113 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=71e4144517084e3087907b27d7a03a66]
14:03:44.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.510986 23113 tablet_loader.cc:96] loaded metadata for tablet 4bd3f6916eba4f2fb1a9aace41281511 (table kudu_system.kudu_transactions [id=71e4144517084e3087907b27d7a03a66])
14:03:44.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.511490 23113 tablet_loader.cc:96] loaded metadata for tablet 79e2cea363a6408d9a53d72158f80fa2 (table txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae])
14:03:44.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.511886 23113 tablet_loader.cc:96] loaded metadata for tablet a382527694d540cb9a48ab998871f9ff (table txn_manager_fallback_rolling [id=08d3968b837044f989a5b012b478ceae])
14:03:44.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.512194 23113 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:03:44.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.513304 23113 catalog_manager.cc:1261] Loaded cluster ID: e33ca72474a94b3695af9e40ba7dd46c
14:03:44.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.513403 23113 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:03:44.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.514740 23113 catalog_manager.cc:1506] Loading token signing keys...
14:03:44.515 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.125:36393(127.21.123.125:36393)] connecting to peer
14:03:44.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.515697 23113 catalog_manager.cc:5966] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73: Loaded TSK: 0
14:03:44.516 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393
14:03:44.516 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.125:36393(127.21.123.125:36393)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393
14:03:44.516 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.21.123.125:36393(127.21.123.125:36393) for tablet Kudu Master: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393
14:03:44.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.516865 23113 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:03:44.535 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.125:36393(127.21.123.125:36393)] connecting to peer
14:03:44.535 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.21.123.126:34049(127.21.123.126:34049)] connecting to peer
14:03:44.536 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393
14:03:44.536 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.123.125:36393(127.21.123.125:36393)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393
14:03:44.536 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.123.125:36393: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393
14:03:44.537 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.123.126:34049(127.21.123.126:34049)] Successfully connected to peer
14:03:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.545038 22590 heartbeater.cc:344] Connected to a master server at 127.21.123.126:34049
14:03:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.545254 22590 heartbeater.cc:499] Master 127.21.123.126:34049 was elected leader, sending a full tablet report...
14:03:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.546792 23037 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8c899ae4122d4a4fa4aa06b02a2ca2d3" instance_seqno: 1750687417461194) as {username='slave'} at 127.21.123.66:52837; Asking this server to re-register.
14:03:44.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.547675 22590 heartbeater.cc:461] Registering TS with master...
14:03:44.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.547897 22590 heartbeater.cc:507] Master 127.21.123.126:34049 requested a full tablet report, sending...
14:03:44.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.549224 23037 ts_manager.cc:194] Registered new tserver with Master: 8c899ae4122d4a4fa4aa06b02a2ca2d3 (127.21.123.66:45807)
14:03:44.556 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x29f30a93, L:/127.0.0.1:47976 - R:/127.21.123.126:34049] using TOKEN/null
14:03:44.557 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:03:44.557 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=42), Traces: [0ms] sending RPC to server master-127.21.123.125:36393, [1ms] received response from server master-127.21.123.125:36393: Network error: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: Network error: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393, [27ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, deferred=Deferred@1176719628(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@1f10a94a, errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@52bc5d78)) after lookup
14:03:44.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:44.587286 23024 consensus_peers.cc:487] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 -> Peer 891a087227fa4198b42224b537877725 (127.21.123.125:36393): Couldn't send request to peer 891a087227fa4198b42224b537877725. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.601976 23037 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47990:
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:03:44.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:44.605034 23083 master.cc:464] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 2 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
14:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.615459 22897 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [term 5 FOLLOWER]: Refusing update from remote peer c51d2fbfbf0a4b38819a021299252b73: Log matching property violated. Preceding OpId in replica: term: 4 index: 13. Preceding OpId from leader: term: 5 index: 14. (index mismatch)
14:03:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.616209 23104 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [LEADER]: Connected to new peer: Peer: permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
14:03:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.617939 23107 sys_catalog.cc:455] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c51d2fbfbf0a4b38819a021299252b73. Latest consensus state: current_term: 5 leader_uuid: "c51d2fbfbf0a4b38819a021299252b73" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.618242 23107 sys_catalog.cc:458] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: This master's current role is: FOLLOWER
14:03:44.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.619374 23087 sys_catalog.cc:455] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "c51d2fbfbf0a4b38819a021299252b73" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:44.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.619580 23087 sys_catalog.cc:458] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 [sys.catalog]: This master's current role is: LEADER
14:03:44.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.620323 23107 sys_catalog.cc:455] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "c51d2fbfbf0a4b38819a021299252b73" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51d2fbfbf0a4b38819a021299252b73" member_type: VOTER last_known_addr { host: "127.21.123.126" port: 34049 } } peers { permanent_uuid: "891a087227fa4198b42224b537877725" member_type: VOTER last_known_addr { host: "127.21.123.125" port: 36393 } } peers { permanent_uuid: "289f4dff4a8f4197a18cdb96452c9887" member_type: VOTER last_known_addr { host: "127.21.123.124" port: 38761 } } }
14:03:44.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.620525 23107 sys_catalog.cc:458] T 00000000000000000000000000000000 P 289f4dff4a8f4197a18cdb96452c9887 [sys.catalog]: This master's current role is: FOLLOWER
14:03:44.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:44.766662 23067 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:47976 (request call id 1) took 208 ms (client timeout 208 ms). Trace:
14:03:44.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:44.766847 23067 rpcz_store.cc:269] 0623 14:03:44.558186 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:03:44.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:44.558242 (+ 56us) service_pool.cc:225] Handling call
14:03:44.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:44.766560 (+208318us) inbound_call.cc:173] Queueing success response
14:03:44.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":489,"init.queue_time_us":43,"thread_start_us":883,"threads_started":4}
14:03:44.768 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=3, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.21.123.125:36393, [1ms] received response from server master-127.21.123.125:36393: Network error: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: Network error: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393, [27ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [42ms] sending RPC to server master-127.21.123.126:34049, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1176719628(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@1f10a94a -> (continuation of Deferred@290273379 after retry RPC@1295691895), errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@52bc5d78 -> (continuation of Deferred@290273379 after retry RPC@1295691895)))
14:03:44.768 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=3, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.21.123.125:36393, [1ms] received response from server master-127.21.123.125:36393: Network error: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.125:36393, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.124:38761, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.123.126:34049, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.125:36393: Network error: Failed to connect to peer master-127.21.123.125:36393(127.21.123.125:36393): Connection refused: /127.21.123.125:36393, [27ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.124:38761: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.123.126:34049: OK, [42ms] sending RPC to server master-127.21.123.126:34049, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1176719628(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@1f10a94a -> (continuation of Deferred@290273379 after retry RPC@1295691895), errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@52bc5d78 -> (continuation of Deferred@290273379 after retry RPC@1295691895)))
14:03:44.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.795068 22449 heartbeater.cc:344] Connected to a master server at 127.21.123.126:34049
14:03:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.796137 23037 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "e6afa1c1602247b682b32d093471e18d" instance_seqno: 1750687416991477) as {username='slave'} at 127.21.123.65:38717; Asking this server to re-register.
14:03:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.796653 22449 heartbeater.cc:461] Registering TS with master...
14:03:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.796875 22449 heartbeater.cc:507] Master 127.21.123.126:34049 requested a full tablet report, sending...
14:03:44.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:44.797971 23037 ts_manager.cc:194] Registered new tserver with Master: e6afa1c1602247b682b32d093471e18d (127.21.123.65:45201)
14:03:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:45.025586 23067 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:47976 (request call id 2) took 250 ms (client timeout 250 ms). Trace:
14:03:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:45.025718 23067 rpcz_store.cc:269] 0623 14:03:44.775323 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:03:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:44.775399 (+ 76us) service_pool.cc:225] Handling call
14:03:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:45.025505 (+250106us) inbound_call.cc:173] Queueing success response
14:03:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:03:45.026 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.123.126:34049, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1467619939(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@679dda69, errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@1feb68d9))
14:03:45.026 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.123.126:34049, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1467619939(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@679dda69, errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@1feb68d9))
14:03:45.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:45.068938 23024 consensus_peers.cc:487] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 -> Peer 891a087227fa4198b42224b537877725 (127.21.123.125:36393): Couldn't send request to peer 891a087227fa4198b42224b537877725. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:03:45.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:45.171962 22616 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.167s from last keepalive heartbeat (effective timeout is 1.000s)
14:03:45.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:45.180637 23128 tablet_replica.cc:1304] Aborted: operation has been aborted: cancelling pending write operations
14:03:45.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:45.188174 23131 tablet_replica.cc:1304] Aborted: operation has been aborted: cancelling pending write operations
14:03:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:45.285336 23067 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:47976 (request call id 3) took 250 ms (client timeout 250 ms). Trace:
14:03:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:45.285468 23067 rpcz_store.cc:269] 0623 14:03:45.035127 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:03:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:45.035216 (+ 89us) service_pool.cc:225] Handling call
14:03:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:45.285273 (+250057us) inbound_call.cc:173] Queueing success response
14:03:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:03:45.286 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.123.126:34049, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@130734683(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@e461c3, errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@9a3bd24))
14:03:45.286 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.123.126:34049, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@130734683(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@e461c3, errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@9a3bd24))
14:03:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:45.545851 23067 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:47976 (request call id 4) took 250 ms (client timeout 250 ms). Trace:
14:03:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:45.545979 23067 rpcz_store.cc:269] 0623 14:03:45.295628 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:03:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:45.295692 (+ 64us) service_pool.cc:225] Handling call
14:03:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0623 14:03:45.545781 (+250089us) inbound_call.cc:173] Queueing success response
14:03:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:03:45.546 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.123.126:34049, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1296740713(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@1c83a299, errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@3f4a22b3))
14:03:45.546 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.21.123.126:34049, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1296740713(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/585998101@1c83a299, errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/978413659@3f4a22b3))
14:03:45.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:45.605310 23024 consensus_peers.cc:487] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 -> Peer 891a087227fa4198b42224b537877725 (127.21.123.125:36393): Couldn't send request to peer 891a087227fa4198b42224b537877725. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:45.624066 23036 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47996:
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:45.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:45.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:03:45.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:03:45.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:45.624910 23036 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:03:45.639 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 0ed313e058b5467f883ba31f1d164b3d (127.21.123.67:44431): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1750687418 last_transition_timestamp: 1750687425
14:03:46.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:46.098810 23024 consensus_peers.cc:487] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 -> Peer 891a087227fa4198b42224b537877725 (127.21.123.125:36393): Couldn't send request to peer 891a087227fa4198b42224b537877725. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
14:03:46.210 [DEBUG - main] (Connection.java:575) [peer master-127.21.123.125:36393(127.21.123.125:36393)] disconnecting while in state TERMINATED
14:03:46.210 [DEBUG - main] (Connection.java:575) [peer master-127.21.123.124:38761(127.21.123.124:38761)] disconnecting while in state READY
14:03:46.210 [DEBUG - main] (Connection.java:575) [peer 0ed313e058b5467f883ba31f1d164b3d(127.21.123.67:44431)] disconnecting while in state READY
14:03:46.210 [DEBUG - main] (Connection.java:575) [peer 8c899ae4122d4a4fa4aa06b02a2ca2d3(127.21.123.66:45807)] disconnecting while in state READY
14:03:46.210 [DEBUG - main] (Connection.java:575) [peer master-127.21.123.126:34049(127.21.123.126:34049)] disconnecting while in state READY
14:03:46.211 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 8c899ae4122d4a4fa4aa06b02a2ca2d3(127.21.123.66:45807)] handling channelInactive
14:03:46.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 8c899ae4122d4a4fa4aa06b02a2ca2d3(127.21.123.66:45807)] cleaning up while in state READY due to: connection closed
14:03:46.212 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:03:46.212 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.123.124:38761(127.21.123.124:38761)] handling channelInactive
14:03:46.212 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.124:38761(127.21.123.124:38761)] cleaning up while in state READY due to: connection closed
14:03:46.212 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 0ed313e058b5467f883ba31f1d164b3d(127.21.123.67:44431)] handling channelInactive
14:03:46.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.212800 21997 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu pid 22290
14:03:46.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 0ed313e058b5467f883ba31f1d164b3d(127.21.123.67:44431)] cleaning up while in state READY due to: connection closed
14:03:46.213 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.123.126:34049(127.21.123.126:34049)] handling channelInactive
14:03:46.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.123.126:34049(127.21.123.126:34049)] cleaning up while in state READY due to: connection closed
14:03:46.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.222321 22443 generic_service.cc:196] Checking for leaks (request via RPC)
14:03:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.228546 22732 heartbeater.cc:499] Master 127.21.123.126:34049 was elected leader, sending a full tablet report...
14:03:46.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.317461 21997 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu with pid 22290
14:03:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.335407 21997 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu pid 22453
14:03:46.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.345595 22584 generic_service.cc:196] Checking for leaks (request via RPC)
14:03:46.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.442742 21997 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu with pid 22453
14:03:46.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.461225 21997 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu pid 22594
14:03:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.467451 22726 generic_service.cc:196] Checking for leaks (request via RPC)
14:03:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.543421 22855 master_runner.cc:186] Error getting master registration for 127.21.123.125:36393: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111),
14:03:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.543602 22855 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:03:46.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:46.584681 23024 consensus_peers.cc:487] T 00000000000000000000000000000000 P c51d2fbfbf0a4b38819a021299252b73 -> Peer 891a087227fa4198b42224b537877725 (127.21.123.125:36393): Couldn't send request to peer 891a087227fa4198b42224b537877725. Status: Network error: Client connection negotiation failed: client connection to 127.21.123.125:36393: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
14:03:46.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.588397 21997 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu with pid 22594
14:03:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.609720 21997 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu pid 23005
14:03:46.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.615715 23077 generic_service.cc:196] Checking for leaks (request via RPC)
14:03:46.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.723213 21997 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu with pid 23005
14:03:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250623 14:03:46.743381 22872 connection.cc:537] server connection from 127.0.0.1:40984 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:03:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.743746 21997 external_mini_cluster.cc:1661] Attempting to check leaks for /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu pid 22855
14:03:46.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.749676 22927 generic_service.cc:196] Checking for leaks (request via RPC)
14:03:46.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250623 14:03:46.851029 21997 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLQ_Kwd/build/asan/bin/kudu with pid 22855
14:03:46.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-23T14:03:46Z chronyd exiting
14:03:47.000 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)