03:14:34.483 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:14:34.483 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskaVXZmI/test-tmp
03:14:34.483 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:14:34.483 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskaVXZmI/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:14:34.488 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:14:34.795 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:14:34.795 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:14:34.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:34.798171 4888 mini_oidc.cc:150] Starting JWKS server
03:14:34.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:34.801347 4888 webserver.cc:492] Webserver started at http://127.0.0.1:46207/ using document root <none> and password file <none>
03:14:34.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:34.801743 4888 mini_oidc.cc:186] Starting OIDC Discovery server
03:14:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:34.803351 4888 webserver.cc:492] Webserver started at http://127.0.0.1:43653/ using document root <none> and password file <none>
03:14:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-18T03:14:34Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:14:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-18T03:14:34Z Disabled control of system clock
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:34.859858 4888 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/wal
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/data
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/logs
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/data/info.pb
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.62:43377
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.62
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46207/jwks.json
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:14:34.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:14:35.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.164548 4900 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:14:35.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.165171 4900 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:14:35.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.165736 4900 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:14:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.197680 4900 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:14:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.198144 4900 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:14:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.198426 4900 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:14:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.198734 4900 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:14:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.199018 4900 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:14:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.199286 4900 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:14:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.199498 4900 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46207/jwks.json
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:35.235718 4900 master_runner.cc:386] Master server non-default flags:
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/data
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/wal
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.62:43377
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46207/jwks.json
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/data/info.pb
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.62
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/logs
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Mar 2026 02:43:20 UTC on e7f111948823
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10983
03:14:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:14:35.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:35.237296 4900 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:14:35.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:35.238950 4900 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:14:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.249451 4906 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:36.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:35.250840 4907 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.347821 4909 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:36.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.349817 4908 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1095 milliseconds
03:14:36.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.349958 4900 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:14:36.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.351204 4900 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:14:36.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.354130 4900 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:14:36.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.355530 4900 hybrid_clock.cc:648] HybridClock initialized: now 1773803676355484 us; error 48 us; skew 500 ppm
03:14:36.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.356345 4900 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:14:36.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.362421 4900 webserver.cc:492] Webserver started at http://127.4.198.62:42387/ using document root <none> and password file <none>
03:14:36.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.363322 4900 fs_manager.cc:362] Metadata directory not provided
03:14:36.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.363660 4900 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:14:36.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.364161 4900 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:14:36.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.368587 4900 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/data/instance:
03:14:36.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79bba6a868c54f17a187e7418fe73c49"
03:14:36.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:36 on dist-test-slave-vs5b"
03:14:36.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.369699 4900 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/wal/instance:
03:14:36.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79bba6a868c54f17a187e7418fe73c49"
03:14:36.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:36 on dist-test-slave-vs5b"
03:14:36.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.376928 4900 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
03:14:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.382848 4916 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.383879 4900 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.384198 4900 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/data,/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/wal
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79bba6a868c54f17a187e7418fe73c49"
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:36 on dist-test-slave-vs5b"
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.384497 4900 fs_report.cc:389] FS layout report
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/wal
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/wal
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/data/data
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:14:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:14:36.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:14:36.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:14:36.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:14:36.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:14:36.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:14:36.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:14:36.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:14:36.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:14:36.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.445500 4900 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:14:36.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.446906 4900 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:14:36.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.447302 4900 kserver.cc:163] Server-wide thread pool size limit: 3276
03:14:36.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.526366 4900 rpc_server.cc:307] RPC server started. Bound to: 127.4.198.62:43377
03:14:36.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.526425 4977 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.198.62:43377 every 8 connection(s)
03:14:36.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.528955 4900 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/data/info.pb
03:14:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.534768 4978 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:36.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.536409 4888 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu as pid 4900
03:14:36.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.536772 4888 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-0/wal/instance
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.539901 4888 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/wal
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/data
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/logs
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/data/info.pb
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:14:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.61:45443
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.61
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46207/jwks.json
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:14:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:14:36.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.545339 4978 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } has no permanent_uuid. Determining permanent_uuid...
03:14:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.570202 4978 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } has no permanent_uuid. Determining permanent_uuid...
03:14:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.572404 4917 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111)
03:14:36.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.576133 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.61:45443: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111)
03:14:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.625571 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } attempt: 1
03:14:36.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.629310 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.61:45443: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111)
03:14:36.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.697675 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } attempt: 2
03:14:36.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.701479 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.61:45443: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111)
03:14:36.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.792833 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } attempt: 3
03:14:36.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.796651 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.61:45443: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111)
03:14:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.835669 4980 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:14:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.836277 4980 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:14:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.836764 4980 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:14:36.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.867691 4980 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:14:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.867965 4980 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:14:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.868158 4980 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:14:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.868371 4980 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:14:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.868563 4980 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:14:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.868757 4980 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:14:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.868901 4980 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46207/jwks.json
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.907845 4980 master_runner.cc:386] Master server non-default flags:
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/data
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/wal
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.61:45443
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46207/jwks.json
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/data/info.pb
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.61
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/logs
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:14:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Mar 2026 02:43:20 UTC on e7f111948823
03:14:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10983
03:14:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:14:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.909113 4980 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:14:36.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.910691 4980 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:14:36.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.921281 4989 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:36.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:36.940029 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } attempt: 4
03:14:36.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.945649 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.61:45443: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111)
03:14:37.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:37.245118 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } attempt: 5
03:14:37.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:37.251698 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.61:45443: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111)
03:14:37.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:37.799171 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } attempt: 6
03:14:37.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:37.827452 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.61:45443: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111)
03:14:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.323770 4988 debug-util.cc:398] Leaking SignalData structure 0x7b08000376a0 after lost signal to thread 4980
03:14:38.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.458724 4988 kernel_stack_watchdog.cc:198] Thread 4980 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
03:14:38.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:14:38.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:14:38.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:14:38.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:14:38.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.459025 4980 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.538s user 0.644s sys 0.894s
03:14:38.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.459394 4980 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.538s user 0.644s sys 0.894s
03:14:38.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:36.921720 4990 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:38.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.461406 4992 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:38.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.464188 4991 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1539 milliseconds
03:14:38.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.464193 4980 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:14:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.465440 4980 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:14:38.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.467839 4980 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:14:38.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.469183 4980 hybrid_clock.cc:648] HybridClock initialized: now 1773803678469135 us; error 54 us; skew 500 ppm
03:14:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.469940 4980 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:14:38.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.475876 4980 webserver.cc:492] Webserver started at http://127.4.198.61:38181/ using document root <none> and password file <none>
03:14:38.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.476778 4980 fs_manager.cc:362] Metadata directory not provided
03:14:38.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.476959 4980 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:14:38.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.477340 4980 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:14:38.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.481469 4980 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/data/instance:
03:14:38.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be6186a03e5f46e685128bf5ce84b5f4"
03:14:38.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:38 on dist-test-slave-vs5b"
03:14:38.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.482462 4980 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/wal/instance:
03:14:38.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be6186a03e5f46e685128bf5ce84b5f4"
03:14:38.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:38 on dist-test-slave-vs5b"
03:14:38.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.489173 4980 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.002s sys 0.006s
03:14:38.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.494472 5001 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:14:38.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.495460 4980 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.001s
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.495735 4980 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/data,/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/wal
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be6186a03e5f46e685128bf5ce84b5f4"
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:38 on dist-test-slave-vs5b"
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.496027 4980 fs_report.cc:389] FS layout report
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/wal
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/wal
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/data/data
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:14:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:14:38.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.544428 4980 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:14:38.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.545799 4980 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:14:38.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.546187 4980 kserver.cc:163] Server-wide thread pool size limit: 3276
03:14:38.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.623868 4980 rpc_server.cc:307] RPC server started. Bound to: 127.4.198.61:45443
03:14:38.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.623935 5062 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.198.61:45443 every 8 connection(s)
03:14:38.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.626565 4980 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/data/info.pb
03:14:38.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.631119 4888 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu as pid 4980
03:14:38.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.631445 4888 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-1/wal/instance
03:14:38.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.632573 5063 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.633419 4888 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/logs
03:14:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data/info.pb
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.60:37911
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.60
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46207/jwks.json
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:14:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:14:38.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.645746 5063 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } has no permanent_uuid. Determining permanent_uuid...
03:14:38.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.669698 5063 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } has no permanent_uuid. Determining permanent_uuid...
03:14:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.683157 5063 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } has no permanent_uuid. Determining permanent_uuid...
03:14:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.685437 5004 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:38.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.689409 5063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:38.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.738831 5063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 1
03:14:38.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.742766 5063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:38.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.811163 5063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 2
03:14:38.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.815086 5063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.887979 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } attempt: 7
03:14:38.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.900931 4978 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } has no permanent_uuid. Determining permanent_uuid...
03:14:38.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.906544 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:38.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.906440 5063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 3
03:14:38.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.910542 5063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:38.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.941983 5065 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:14:38.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.942600 5065 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:14:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.943161 5065 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:14:38.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:38.964968 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 1
03:14:38.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.968827 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:38.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.975242 5065 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:14:38.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.975567 5065 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:14:38.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.975805 5065 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:14:38.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.976064 5065 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:14:38.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.976385 5065 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:14:38.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.976692 5065 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:14:38.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:38.976897 5065 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46207/jwks.json
03:14:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:39.012439 5065 master_runner.cc:386] Master server non-default flags:
03:14:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.60:37911
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46207/jwks.json
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data/info.pb
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.60
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/logs
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Mar 2026 02:43:20 UTC on e7f111948823
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10983
03:14:39.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:14:39.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:39.013968 5065 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:14:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:39.015512 5065 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:14:39.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:39.025548 5077 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:39.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:39.050218 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 2
03:14:39.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:39.053929 5063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 4
03:14:39.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:39.059906 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:39.062371 5063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:39.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:39.148197 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 3
03:14:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:39.157109 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:39.297575 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 4
03:14:39.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:39.303654 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:39.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:39.361805 5063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 5
03:14:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:39.367939 5063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:39.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:39.587134 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 5
03:14:39.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:39.593219 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:39.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:39.915433 5063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 6
03:14:39.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:39.937422 5063 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:40.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.145714 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 6
03:14:40.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:40.170262 4978 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.198.60:37911: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:40.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:40.428927 5076 debug-util.cc:398] Leaking SignalData structure 0x7b08000376a0 after lost signal to thread 5065
03:14:40.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:39.026041 5078 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:40.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:40.492679 5065 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.467s user 0.001s sys 0.000s
03:14:40.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:40.493053 5065 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.467s user 0.002s sys 0.000s
03:14:40.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:40.494894 5080 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:40.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:40.496903 5079 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1468 milliseconds
03:14:40.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.496984 5065 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:14:40.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.498171 5065 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:14:40.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.500813 5065 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:14:40.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.502218 5065 hybrid_clock.cc:648] HybridClock initialized: now 1773803680502174 us; error 49 us; skew 500 ppm
03:14:40.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.502981 5065 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:14:40.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.508698 5065 webserver.cc:492] Webserver started at http://127.4.198.60:37305/ using document root <none> and password file <none>
03:14:40.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.509546 5065 fs_manager.cc:362] Metadata directory not provided
03:14:40.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.509725 5065 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:14:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.510113 5065 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:14:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.514263 5065 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data/instance:
03:14:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cfc16134dcdc440a843537d928cf2c3e"
03:14:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:40 on dist-test-slave-vs5b"
03:14:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.515254 5065 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal/instance:
03:14:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cfc16134dcdc440a843537d928cf2c3e"
03:14:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:40 on dist-test-slave-vs5b"
03:14:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.522226 5065 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
03:14:40.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.528304 5091 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:14:40.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.529587 5065 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:14:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.529940 5065 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data,/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal
03:14:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cfc16134dcdc440a843537d928cf2c3e"
03:14:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:40 on dist-test-slave-vs5b"
03:14:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.530310 5065 fs_report.cc:389] FS layout report
03:14:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:14:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal
03:14:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data/data
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:14:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:14:40.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.579489 5065 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:14:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.581009 5065 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:14:40.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.581480 5065 kserver.cc:163] Server-wide thread pool size limit: 3276
03:14:40.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.662437 5065 rpc_server.cc:307] RPC server started. Bound to: 127.4.198.60:37911
03:14:40.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.662516 5152 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.198.60:37911 every 8 connection(s)
03:14:40.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.665066 5065 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data/info.pb
03:14:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.669862 4888 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu as pid 5065
03:14:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.670333 4888 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal/instance
03:14:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.671254 5153 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:40.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.686259 5153 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } has no permanent_uuid. Determining permanent_uuid...
03:14:40.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.708566 5153 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } has no permanent_uuid. Determining permanent_uuid...
03:14:40.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.717949 5153 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } has no permanent_uuid. Determining permanent_uuid...
03:14:40.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.732831 5153 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: Bootstrap starting.
03:14:40.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.738427 5153 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: Neither blocks nor log segments found. Creating new log.
03:14:40.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.740079 5153 log.cc:826] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: Log is configured to *not* fsync() on all Append() calls
03:14:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.744079 5153 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: No bootstrap required, opened a new log
03:14:40.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.762517 5153 raft_consensus.cc:359] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:40.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.763142 5153 raft_consensus.cc:385] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:40.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.763422 5153 raft_consensus.cc:740] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cfc16134dcdc440a843537d928cf2c3e, State: Initialized, Role: FOLLOWER
03:14:40.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.764355 5153 consensus_queue.cc:260] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.767154 5162 sys_catalog.cc:455] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.767696 5162 sys_catalog.cc:458] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: This master's current role is: FOLLOWER
03:14:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.768623 5153 sys_catalog.cc:565] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: configured and running, proceeding with master startup.
03:14:40.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.774418 5162 raft_consensus.cc:493] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:14:40.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.774818 5162 raft_consensus.cc:515] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:40.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.777292 5162 leader_election.cc:290] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79bba6a868c54f17a187e7418fe73c49 (127.4.198.62:43377), be6186a03e5f46e685128bf5ce84b5f4 (127.4.198.61:45443)
03:14:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:40.795112 5173 catalog_manager.cc:1568] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:14:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:40.795442 5173 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:14:40.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.790915 5065 master_runner.cc:186] Error getting master registration for 127.4.198.62:43377: OK, instance_id { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" instance_seqno: 1773803676508567 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:14:40.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.821300 5065 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:14:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:40.836476 5097 tablet.cc:2379] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:14:40.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:40.997915 5063 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 7
03:14:41.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.020409 5063 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4: Bootstrap starting.
03:14:41.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.027962 5063 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4: Neither blocks nor log segments found. Creating new log.
03:14:41.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.030467 5063 log.cc:826] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4: Log is configured to *not* fsync() on all Append() calls
03:14:41.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.035621 5063 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4: No bootstrap required, opened a new log
03:14:41.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.063122 5063 raft_consensus.cc:359] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:41.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.064023 5063 raft_consensus.cc:385] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:41.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.064360 5063 raft_consensus.cc:740] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: be6186a03e5f46e685128bf5ce84b5f4, State: Initialized, Role: FOLLOWER
03:14:41.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.065438 5063 consensus_queue.cc:260] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:41.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.068163 5177 sys_catalog.cc:455] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:41.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.068910 5177 sys_catalog.cc:458] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [sys.catalog]: This master's current role is: FOLLOWER
03:14:41.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.070151 5063 sys_catalog.cc:565] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [sys.catalog]: configured and running, proceeding with master startup.
03:14:41.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.070343 5028 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cfc16134dcdc440a843537d928cf2c3e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "be6186a03e5f46e685128bf5ce84b5f4" is_pre_election: true
03:14:41.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.071229 5028 raft_consensus.cc:2468] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cfc16134dcdc440a843537d928cf2c3e in term 0.
03:14:41.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.075435 5092 leader_election.cc:304] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: be6186a03e5f46e685128bf5ce84b5f4, cfc16134dcdc440a843537d928cf2c3e; no voters:
03:14:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.077314 5162 raft_consensus.cc:2804] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 0 FOLLOWER]: Leader pre-election won for term 1
03:14:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.077665 5162 raft_consensus.cc:493] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:14:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.078001 5162 raft_consensus.cc:3060] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 0 FOLLOWER]: Advancing to term 1
03:14:41.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.084725 5162 raft_consensus.cc:515] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:41.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.086558 5162 leader_election.cc:290] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [CANDIDATE]: Term 1 election: Requested vote from peers 79bba6a868c54f17a187e7418fe73c49 (127.4.198.62:43377), be6186a03e5f46e685128bf5ce84b5f4 (127.4.198.61:45443)
03:14:41.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.088392 5028 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cfc16134dcdc440a843537d928cf2c3e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "be6186a03e5f46e685128bf5ce84b5f4"
03:14:41.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.088966 5028 raft_consensus.cc:3060] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 0 FOLLOWER]: Advancing to term 1
03:14:41.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.097394 5028 raft_consensus.cc:2468] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cfc16134dcdc440a843537d928cf2c3e in term 1.
03:14:41.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.098611 5092 leader_election.cc:304] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: be6186a03e5f46e685128bf5ce84b5f4, cfc16134dcdc440a843537d928cf2c3e; no voters:
03:14:41.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.099457 5162 raft_consensus.cc:2804] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 1 FOLLOWER]: Leader election won for term 1
03:14:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.115872 5162 raft_consensus.cc:697] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 1 LEADER]: Becoming Leader. State: Replica: cfc16134dcdc440a843537d928cf2c3e, State: Running, Role: LEADER
03:14:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.117098 5162 consensus_queue.cc:237] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:41.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.146978 5186 sys_catalog.cc:455] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: SysCatalogTable state changed. Reason: New leader cfc16134dcdc440a843537d928cf2c3e. Latest consensus state: current_term: 1 leader_uuid: "cfc16134dcdc440a843537d928cf2c3e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:41.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.147744 5186 sys_catalog.cc:458] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: This master's current role is: LEADER
03:14:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:41.151193 5189 catalog_manager.cc:1568] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:14:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:41.151532 5189 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:14:41.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.156773 5192 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:14:41.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.122582 4980 master_runner.cc:186] Error getting master registration for 127.4.198.62:43377: OK, instance_id { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" instance_seqno: 1773803676508567 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:14:41.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.159627 4980 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:14:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.163961 5192 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:14:41.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.188512 5028 raft_consensus.cc:1275] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 1 FOLLOWER]: Refusing update from remote peer cfc16134dcdc440a843537d928cf2c3e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:14:41.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.191676 5186 consensus_queue.cc:1048] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [LEADER]: Connected to new peer: Peer: permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:14:41.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.203719 4978 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } attempt: 7
03:14:41.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:41.219002 5092 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.4.198.62:43377 timed out after 0.441s (SENT)
03:14:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.223155 5177 sys_catalog.cc:455] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cfc16134dcdc440a843537d928cf2c3e. Latest consensus state: current_term: 1 leader_uuid: "cfc16134dcdc440a843537d928cf2c3e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.227638 5177 sys_catalog.cc:458] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [sys.catalog]: This master's current role is: FOLLOWER
03:14:41.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.234520 5193 mvcc.cc:204] Tried to move back new op lower bound from 7265499878132477952 to 7265499877866151936. Current Snapshot: MvccSnapshot[applied={T|T < 7265499878132477952}]
03:14:41.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.254514 5177 sys_catalog.cc:455] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cfc16134dcdc440a843537d928cf2c3e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:41.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.255198 5177 sys_catalog.cc:458] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [sys.catalog]: This master's current role is: FOLLOWER
03:14:41.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.261113 5192 catalog_manager.cc:1357] Generated new cluster ID: 5aff527ef8e1466fb0288a917ecfa345
03:14:41.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.261415 5192 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:14:41.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.262336 5186 sys_catalog.cc:455] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cfc16134dcdc440a843537d928cf2c3e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:41.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.262979 5186 sys_catalog.cc:458] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: This master's current role is: LEADER
03:14:41.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.278465 4978 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49: Bootstrap starting.
03:14:41.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.285604 4978 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49: Neither blocks nor log segments found. Creating new log.
03:14:41.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.287963 4978 log.cc:826] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49: Log is configured to *not* fsync() on all Append() calls
03:14:41.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.290081 5192 catalog_manager.cc:1380] Generated new certificate authority record
03:14:41.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.299546 4978 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49: No bootstrap required, opened a new log
03:14:41.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.307158 5192 catalog_manager.cc:1514] Loading token signing keys...
03:14:41.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.328738 4978 raft_consensus.cc:359] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:41.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.329749 4978 raft_consensus.cc:385] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:41.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.330181 4978 raft_consensus.cc:740] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 79bba6a868c54f17a187e7418fe73c49, State: Initialized, Role: FOLLOWER
03:14:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.331249 4978 consensus_queue.cc:260] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.345347 4942 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cfc16134dcdc440a843537d928cf2c3e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "79bba6a868c54f17a187e7418fe73c49"
03:14:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.345821 4940 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cfc16134dcdc440a843537d928cf2c3e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "79bba6a868c54f17a187e7418fe73c49" is_pre_election: true
03:14:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.345340 4943 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cfc16134dcdc440a843537d928cf2c3e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "79bba6a868c54f17a187e7418fe73c49" is_pre_election: true
03:14:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.346742 5201 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.348085 5201 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: This master's current role is: FOLLOWER
03:14:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.347698 4978 sys_catalog.cc:565] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: configured and running, proceeding with master startup.
03:14:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.354295 4941 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 0 FOLLOWER]: Advancing to term 1
03:14:41.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:41.358840 4942 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:42248 (request call id 3) took 271 ms (client timeout 361 ms). Trace:
03:14:41.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:41.359189 4942 rpcz_store.cc:269] 0318 03:14:41.087314 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:14:41.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0318 03:14:41.087714 (+ 400us) service_pool.cc:225] Handling call
03:14:41.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0318 03:14:41.358053 (+270339us) spinlock_profiling.cc:243] Waited 1.6 ms on lock 0x7b5800001500. stack: 00007f4d7a221423 00007f4d7a221320 00007f4d79d96574 0000000000509544 00007f4d7a0afd71 00007f4d7a0baa6a 00007f4d7a0b93b9 00007f4d81317040 00007f4d82343bd7 00007f4d809fb834 00007f4d809fb7c2 00007f4d809fb6f1 00007f4d809fb66c 00007f4d809fa922 00007f4d7f00f34c 00007f4d7f00e786
03:14:41.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0318 03:14:41.358757 (+ 704us) inbound_call.cc:173] Queueing success response
03:14:41.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":3690240}
03:14:41.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:41.361303 4943 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:42248 (request call id 1) took 583 ms (client timeout 441 ms). Trace:
03:14:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:41.361625 4943 rpcz_store.cc:269] 0318 03:14:40.778212 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:14:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0318 03:14:40.778336 (+ 124us) service_pool.cc:225] Handling call
03:14:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0318 03:14:41.360912 (+582576us) spinlock_profiling.cc:243] Waited 3.35 ms on lock 0x7b5800001500. stack: 00007f4d7a221423 00007f4d7a221320 00007f4d79d96574 0000000000509544 00007f4d7a0afd71 00007f4d7a0baa6a 00007f4d7a0b93b9 00007f4d81317040 00007f4d82343bd7 00007f4d809fb834 00007f4d809fb7c2 00007f4d809fb6f1 00007f4d809fb66c 00007f4d809fa922 00007f4d7f00f34c 00007f4d7f00e786
03:14:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0318 03:14:41.361232 (+ 320us) inbound_call.cc:173] Queueing success response
03:14:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":7710464}
03:14:41.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.374715 4941 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 1 FOLLOWER]: Refusing update from remote peer cfc16134dcdc440a843537d928cf2c3e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:14:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.379480 5162 consensus_queue.cc:1048] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [LEADER]: Connected to new peer: Peer: permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:14:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.432767 5192 catalog_manager.cc:6044] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: Generated new TSK 0
03:14:41.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.435851 5192 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:14:41.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.471222 5201 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cfc16134dcdc440a843537d928cf2c3e. Latest consensus state: current_term: 1 leader_uuid: "cfc16134dcdc440a843537d928cf2c3e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:41.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.471896 5201 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: This master's current role is: FOLLOWER
03:14:41.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.473107 5211 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cfc16134dcdc440a843537d928cf2c3e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:41.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.473922 5211 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: This master's current role is: FOLLOWER
03:14:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.472076 5162 sys_catalog.cc:455] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cfc16134dcdc440a843537d928cf2c3e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.476881 5162 sys_catalog.cc:458] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: This master's current role is: LEADER
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.487015 4888 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/wal
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/data
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/logs
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/data/info.pb
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.1:0
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.198.1
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.1
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:41.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:14:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:14:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:14:41.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.504274 5218 catalog_manager.cc:1269] Loaded cluster ID: 5aff527ef8e1466fb0288a917ecfa345
03:14:41.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.504763 5218 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49: loading cluster ID for follower catalog manager: success
03:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.511833 5218 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49: acquiring CA information for follower catalog manager: success
03:14:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:41.516770 5218 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:14:42.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:42.154078 5220 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:14:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:42.154965 5220 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:14:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:42.156138 5220 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:14:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:42.156785 5189 catalog_manager.cc:1269] Loaded cluster ID: 5aff527ef8e1466fb0288a917ecfa345
03:14:42.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:42.157054 5189 catalog_manager.cc:1562] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4: loading cluster ID for follower catalog manager: success
03:14:42.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:42.163733 5189 catalog_manager.cc:1584] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4: acquiring CA information for follower catalog manager: success
03:14:42.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:42.181226 5189 catalog_manager.cc:1612] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:14:42.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:42.215979 5220 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:14:42.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:42.217154 5220 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:14:42.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:42.217602 5220 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:14:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:42.217967 5220 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:14:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:42.218791 5220 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.198.1
03:14:42.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:42.284505 5220 tablet_server_runner.cc:78] Tablet server non-default flags:
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/data
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/wal
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.1:0
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/data/info.pb
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.1
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.198.1
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/logs
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Mar 2026 02:43:20 UTC on e7f111948823
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10983
03:14:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:14:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:42.287133 5220 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:14:42.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:42.289471 5220 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:14:42.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:42.324203 5226 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:44.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:43.717850 5225 debug-util.cc:398] Leaking SignalData structure 0x7b0800028140 after lost signal to thread 5220
03:14:44.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:44.466434 5220 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.139s user 0.569s sys 0.717s
03:14:44.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:42.329691 5227 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:44.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:44.466755 5220 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.139s user 0.569s sys 0.717s
03:14:44.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:44.474200 5231 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:44.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.474233 5220 server_base.cc:1060] running on GCE node
03:14:44.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.475411 5220 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:14:44.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.477542 5220 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:14:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.478907 5220 hybrid_clock.cc:648] HybridClock initialized: now 1773803684478873 us; error 28 us; skew 500 ppm
03:14:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.479781 5220 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:14:44.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.485718 5220 webserver.cc:492] Webserver started at http://127.4.198.1:44839/ using document root <none> and password file <none>
03:14:44.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.486604 5220 fs_manager.cc:362] Metadata directory not provided
03:14:44.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.486830 5220 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:14:44.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.487287 5220 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:14:44.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.491847 5220 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/data/instance:
03:14:44.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "823feaa369b047a89e2b9bdfaa6f3ca9"
03:14:44.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:44 on dist-test-slave-vs5b"
03:14:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.493013 5220 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/wal/instance:
03:14:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "823feaa369b047a89e2b9bdfaa6f3ca9"
03:14:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:44 on dist-test-slave-vs5b"
03:14:44.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.500186 5220 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
03:14:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.505702 5237 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:14:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.506731 5220 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
03:14:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.507086 5220 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/data,/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/wal
03:14:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "823feaa369b047a89e2b9bdfaa6f3ca9"
03:14:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:44 on dist-test-slave-vs5b"
03:14:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.507436 5220 fs_report.cc:389] FS layout report
03:14:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/wal
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/wal
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/data/data
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:14:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:14:44.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.569334 5220 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:14:44.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.570940 5220 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:14:44.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.571404 5220 kserver.cc:163] Server-wide thread pool size limit: 3276
03:14:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.580672 5220 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:14:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.580902 5220 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:14:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.581188 5220 ts_tablet_manager.cc:616] Registered 0 tablets
03:14:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.581357 5220 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:14:44.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.763273 5220 rpc_server.cc:307] RPC server started. Bound to: 127.4.198.1:35799
03:14:44.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.763777 5362 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.198.1:35799 every 8 connection(s)
03:14:44.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.766008 5220 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/data/info.pb
03:14:44.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.767141 4888 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu as pid 5220
03:14:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.767828 4888 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-0/wal/instance
03:14:44.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.783229 5365 heartbeater.cc:344] Connected to a master server at 127.4.198.62:43377
03:14:44.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.783706 5365 heartbeater.cc:461] Registering TS with master...
03:14:44.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.785046 5365 heartbeater.cc:507] Master 127.4.198.62:43377 requested a full tablet report, sending...
03:14:44.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.788303 4933 ts_manager.cc:194] Registered new tserver with Master: 823feaa369b047a89e2b9bdfaa6f3ca9 (127.4.198.1:35799)
03:14:44.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.788872 5363 heartbeater.cc:344] Connected to a master server at 127.4.198.60:37911
03:14:44.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.789194 5363 heartbeater.cc:461] Registering TS with master...
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.790025 5363 heartbeater.cc:507] Master 127.4.198.60:37911 requested a full tablet report, sending...
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.791541 4888 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/wal
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/data
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/logs
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/data/info.pb
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.2:0
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.198.2
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.2
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:14:44.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.792714 5107 ts_manager.cc:194] Registered new tserver with Master: 823feaa369b047a89e2b9bdfaa6f3ca9 (127.4.198.1:35799)
03:14:44.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.794833 5107 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.198.1:56001
03:14:44.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.798451 5364 heartbeater.cc:344] Connected to a master server at 127.4.198.61:45443
03:14:44.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.798740 5364 heartbeater.cc:461] Registering TS with master...
03:14:44.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.799269 5364 heartbeater.cc:507] Master 127.4.198.61:45443 requested a full tablet report, sending...
03:14:44.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:44.801533 5018 ts_manager.cc:194] Registered new tserver with Master: 823feaa369b047a89e2b9bdfaa6f3ca9 (127.4.198.1:35799)
03:14:45.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:45.089766 5368 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:14:45.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:45.090216 5368 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:14:45.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:45.090848 5368 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:14:45.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:45.123823 5368 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:14:45.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:45.124433 5368 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:14:45.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:45.124622 5368 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:14:45.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:45.124784 5368 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:14:45.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:45.125172 5368 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.198.2
03:14:45.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:45.163466 5368 tablet_server_runner.cc:78] Tablet server non-default flags:
03:14:45.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:45.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:45.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:45.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:45.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/data
03:14:45.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/wal
03:14:45.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:45.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.2:0
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/data/info.pb
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.2
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.198.2
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/logs
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Mar 2026 02:43:20 UTC on e7f111948823
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10983
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:14:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:45.164740 5368 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:14:45.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:45.166286 5368 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:14:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:45.181866 5375 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:45.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:45.798516 5363 heartbeater.cc:499] Master 127.4.198.60:37911 was elected leader, sending a full tablet report...
03:14:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:45.187130 5378 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:45.182938 5376 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:46.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:46.330363 5377 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1143 milliseconds
03:14:46.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.330472 5368 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:14:46.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.331676 5368 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:14:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.334208 5368 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:14:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.335599 5368 hybrid_clock.cc:648] HybridClock initialized: now 1773803686335559 us; error 54 us; skew 500 ppm
03:14:46.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.336390 5368 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:14:46.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.342458 5368 webserver.cc:492] Webserver started at http://127.4.198.2:39167/ using document root <none> and password file <none>
03:14:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.343308 5368 fs_manager.cc:362] Metadata directory not provided
03:14:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.343483 5368 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:14:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.343859 5368 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:14:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.348115 5368 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/data/instance:
03:14:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b6e1325a34b4f2c88e8626a3c208d6e"
03:14:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:46 on dist-test-slave-vs5b"
03:14:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.349169 5368 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/wal/instance:
03:14:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b6e1325a34b4f2c88e8626a3c208d6e"
03:14:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:46 on dist-test-slave-vs5b"
03:14:46.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.356482 5368 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
03:14:46.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.361610 5386 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:14:46.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.362675 5368 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.362946 5368 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/data,/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/wal
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b6e1325a34b4f2c88e8626a3c208d6e"
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:46 on dist-test-slave-vs5b"
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.363250 5368 fs_report.cc:389] FS layout report
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/wal
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/wal
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/data/data
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:14:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:14:46.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.405155 5368 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:14:46.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.406514 5368 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:14:46.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.406894 5368 kserver.cc:163] Server-wide thread pool size limit: 3276
03:14:46.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.416246 5368 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:14:46.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.416465 5368 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:14:46.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.416756 5368 ts_tablet_manager.cc:616] Registered 0 tablets
03:14:46.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.416919 5368 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:14:46.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.576325 5368 rpc_server.cc:307] RPC server started. Bound to: 127.4.198.2:39509
03:14:46.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.576792 5510 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.198.2:39509 every 8 connection(s)
03:14:46.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.578949 5368 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/data/info.pb
03:14:46.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.587967 4888 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu as pid 5368
03:14:46.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.588510 4888 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-1/wal/instance
03:14:46.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.588747 5513 heartbeater.cc:344] Connected to a master server at 127.4.198.62:43377
03:14:46.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.589200 5513 heartbeater.cc:461] Registering TS with master...
03:14:46.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.590502 5513 heartbeater.cc:507] Master 127.4.198.62:43377 requested a full tablet report, sending...
03:14:46.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.593532 4933 ts_manager.cc:194] Registered new tserver with Master: 5b6e1325a34b4f2c88e8626a3c208d6e (127.4.198.2:39509)
03:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.599866 5511 heartbeater.cc:344] Connected to a master server at 127.4.198.60:37911
03:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.600275 5511 heartbeater.cc:461] Registering TS with master...
03:14:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.601228 5511 heartbeater.cc:507] Master 127.4.198.60:37911 requested a full tablet report, sending...
03:14:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.603392 5107 ts_manager.cc:194] Registered new tserver with Master: 5b6e1325a34b4f2c88e8626a3c208d6e (127.4.198.2:39509)
03:14:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.603401 4888 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/wal
03:14:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/data
03:14:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:14:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:14:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:14:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/logs
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/data/info.pb
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.3:0
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.198.3
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.3
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:14:46.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.605078 5107 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.198.2:59137
03:14:46.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.605185 5512 heartbeater.cc:344] Connected to a master server at 127.4.198.61:45443
03:14:46.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.605587 5512 heartbeater.cc:461] Registering TS with master...
03:14:46.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.606417 5512 heartbeater.cc:507] Master 127.4.198.61:45443 requested a full tablet report, sending...
03:14:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.609007 5018 ts_manager.cc:194] Registered new tserver with Master: 5b6e1325a34b4f2c88e8626a3c208d6e (127.4.198.2:39509)
03:14:46.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:46.905042 5516 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:14:46.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:46.905552 5516 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:14:46.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:46.906196 5516 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:14:46.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:46.940275 5516 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:14:46.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:46.940927 5516 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:14:46.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:46.941167 5516 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:14:46.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:46.941390 5516 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:14:46.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:46.941869 5516 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.198.3
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.976780 5516 tablet_server_runner.cc:78] Tablet server non-default flags:
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/data
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/wal
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.3:0
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/data/info.pb
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.3
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.198.3
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/logs
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:14:46.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Mar 2026 02:43:20 UTC on e7f111948823
03:14:46.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10983
03:14:46.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:14:46.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.978332 5516 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:14:46.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:46.979918 5516 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:14:46.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:46.991158 5523 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:47.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:47.608659 5511 heartbeater.cc:499] Master 127.4.198.60:37911 was elected leader, sending a full tablet report...
03:14:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:48.394191 5522 debug-util.cc:398] Leaking SignalData structure 0x7b0800028140 after lost signal to thread 5516
03:14:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:46.992522 5524 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:48.499372 5522 kernel_stack_watchdog.cc:198] Thread 5516 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
03:14:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:14:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:14:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:14:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:14:48.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:48.499897 5516 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.508s user 0.591s sys 0.904s
03:14:48.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:48.500272 5516 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.508s user 0.591s sys 0.904s
03:14:48.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:48.501861 5526 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:48.504734 5525 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1512 milliseconds
03:14:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.504739 5516 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:14:48.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.505908 5516 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:14:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.507828 5516 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:14:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.509187 5516 hybrid_clock.cc:648] HybridClock initialized: now 1773803688509134 us; error 50 us; skew 500 ppm
03:14:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.509980 5516 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:14:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.515836 5516 webserver.cc:492] Webserver started at http://127.4.198.3:44479/ using document root <none> and password file <none>
03:14:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.516876 5516 fs_manager.cc:362] Metadata directory not provided
03:14:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.517096 5516 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:14:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.517570 5516 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:14:48.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.521915 5516 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/data/instance:
03:14:48.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27a6991a885b47e594fb08f14be5a0f0"
03:14:48.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:48 on dist-test-slave-vs5b"
03:14:48.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.523077 5516 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/wal/instance:
03:14:48.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27a6991a885b47e594fb08f14be5a0f0"
03:14:48.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:48 on dist-test-slave-vs5b"
03:14:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.530557 5516 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
03:14:48.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.536386 5535 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:14:48.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.537535 5516 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
03:14:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.537873 5516 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/data,/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/wal
03:14:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27a6991a885b47e594fb08f14be5a0f0"
03:14:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:48 on dist-test-slave-vs5b"
03:14:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.538196 5516 fs_report.cc:389] FS layout report
03:14:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:14:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/wal
03:14:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/wal
03:14:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/data/data
03:14:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:14:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:14:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:14:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:14:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:14:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:14:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:14:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:14:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:14:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:14:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:14:48.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.582425 5516 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:14:48.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.583884 5516 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:14:48.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.584370 5516 kserver.cc:163] Server-wide thread pool size limit: 3276
03:14:48.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.593572 5516 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:14:48.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.593879 5516 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:14:48.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.594187 5516 ts_tablet_manager.cc:616] Registered 0 tablets
03:14:48.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.594403 5516 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:14:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.793435 5516 rpc_server.cc:307] RPC server started. Bound to: 127.4.198.3:42651
03:14:48.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.794027 5659 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.198.3:42651 every 8 connection(s)
03:14:48.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.795885 5516 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/data/info.pb
03:14:48.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.798195 4888 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu as pid 5516
03:14:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.798650 4888 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/ts-2/wal/instance
03:14:48.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.806391 5662 heartbeater.cc:344] Connected to a master server at 127.4.198.62:43377
03:14:48.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.806857 5662 heartbeater.cc:461] Registering TS with master...
03:14:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.808213 5662 heartbeater.cc:507] Master 127.4.198.62:43377 requested a full tablet report, sending...
03:14:48.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.813014 4933 ts_manager.cc:194] Registered new tserver with Master: 27a6991a885b47e594fb08f14be5a0f0 (127.4.198.3:42651)
03:14:48.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.819020 5660 heartbeater.cc:344] Connected to a master server at 127.4.198.60:37911
03:14:48.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.819401 5660 heartbeater.cc:461] Registering TS with master...
03:14:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.820323 5660 heartbeater.cc:507] Master 127.4.198.60:37911 requested a full tablet report, sending...
03:14:48.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.822293 5108 ts_manager.cc:194] Registered new tserver with Master: 27a6991a885b47e594fb08f14be5a0f0 (127.4.198.3:42651)
03:14:48.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.823637 5661 heartbeater.cc:344] Connected to a master server at 127.4.198.61:45443
03:14:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.823791 5108 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.198.3:43893
03:14:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.823941 5661 heartbeater.cc:461] Registering TS with master...
03:14:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.824688 5661 heartbeater.cc:507] Master 127.4.198.61:45443 requested a full tablet report, sending...
03:14:48.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.826845 5017 ts_manager.cc:194] Registered new tserver with Master: 27a6991a885b47e594fb08f14be5a0f0 (127.4.198.3:42651)
03:14:48.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.835275 4888 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
03:14:48.845 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:14:48.845 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:14:48.846 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.198.62"
port: 43377
}
bound_http_address {
host: "127.4.198.62"
port: 42387
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.198.61"
port: 45443
}
bound_http_address {
host: "127.4.198.61"
port: 38181
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.198.60"
port: 37911
}
bound_http_address {
host: "127.4.198.60"
port: 37305
}
}
}
03:14:48.847 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:14:48.847 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.4.198.1"
port: 35799
}
bound_http_address {
host: "127.4.198.1"
port: 44839
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.4.198.2"
port: 39509
}
bound_http_address {
host: "127.4.198.2"
port: 39167
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.4.198.3"
port: 42651
}
bound_http_address {
host: "127.4.198.3"
port: 44479
}
}
}
03:14:48.847 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:14:48.850 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:14:48.851 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.198.60:37911(127.4.198.60:37911)] connecting to peer
03:14:48.853 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.198.62:43377(127.4.198.62:43377)] connecting to peer
03:14:48.854 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:48.854 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.198.60:37911(127.4.198.60:37911)] Successfully connected to peer
03:14:48.855 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.4.198.61:45443(127.4.198.61:45443)] Successfully connected to peer
03:14:48.855 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.4.198.62:43377(127.4.198.62:43377)] Successfully connected to peer
03:14:48.857 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.4.198.62
03:14:48.857 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.4.198.61
03:14:48.857 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.4.198.60
03:14:48.869 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x79e06d4b, L:/127.0.0.1:39910 - R:/127.4.198.62:43377] using SASL/PLAIN
03:14:48.869 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xdb8b33f3, L:/127.0.0.1:54212 - R:/127.4.198.60:37911] using SASL/PLAIN
03:14:48.871 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x2b20b398, L:/127.0.0.1:53416 - R:/127.4.198.61:45443] using SASL/PLAIN
03:14:48.875 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:14:48.875 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=24), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [20ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: OK, deferred=null) after lookup
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.876576 5108 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54212:
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:14:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:48.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:48.879736 5108 catalog_manager.cc:7033] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:14:48.923 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 38d0a76607c74e47b30cf514a7631d2b
03:14:48.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.989320 5445 tablet_service.cc:1511] Processing CreateTablet for tablet 8d77b3eaef1d44f8842ab9fd4ce8b99d (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:14:48.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.989293 5594 tablet_service.cc:1511] Processing CreateTablet for tablet 8d77b3eaef1d44f8842ab9fd4ce8b99d (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:14:48.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.990005 5595 tablet_service.cc:1511] Processing CreateTablet for tablet 66bcb4903dd74d08a0a8d147498e899b (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:14:48.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.991186 5594 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8d77b3eaef1d44f8842ab9fd4ce8b99d. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:48.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.990618 5446 tablet_service.cc:1511] Processing CreateTablet for tablet 66bcb4903dd74d08a0a8d147498e899b (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:14:48.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.990844 5296 tablet_service.cc:1511] Processing CreateTablet for tablet 8d77b3eaef1d44f8842ab9fd4ce8b99d (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:14:48.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.991993 5297 tablet_service.cc:1511] Processing CreateTablet for tablet 66bcb4903dd74d08a0a8d147498e899b (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:14:48.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.992399 5446 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 66bcb4903dd74d08a0a8d147498e899b. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:48.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.993404 5297 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 66bcb4903dd74d08a0a8d147498e899b. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:48.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.994421 5296 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8d77b3eaef1d44f8842ab9fd4ce8b99d. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:48.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.995002 5445 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8d77b3eaef1d44f8842ab9fd4ce8b99d. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:48.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:48.995064 5595 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 66bcb4903dd74d08a0a8d147498e899b. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:49.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.023072 5689 tablet_bootstrap.cc:492] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9: Bootstrap starting.
03:14:49.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.029799 5688 tablet_bootstrap.cc:492] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e: Bootstrap starting.
03:14:49.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.033982 5689 tablet_bootstrap.cc:654] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9: Neither blocks nor log segments found. Creating new log.
03:14:49.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.036592 5689 log.cc:826] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9: Log is configured to *not* fsync() on all Append() calls
03:14:49.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.039934 5688 tablet_bootstrap.cc:654] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e: Neither blocks nor log segments found. Creating new log.
03:14:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.041221 5690 tablet_bootstrap.cc:492] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0: Bootstrap starting.
03:14:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.042011 5689 tablet_bootstrap.cc:492] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9: No bootstrap required, opened a new log
03:14:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.042382 5689 ts_tablet_manager.cc:1403] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9: Time spent bootstrapping tablet: real 0.020s user 0.013s sys 0.000s
03:14:49.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.046208 5688 log.cc:826] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e: Log is configured to *not* fsync() on all Append() calls
03:14:49.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.046545 5690 tablet_bootstrap.cc:654] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0: Neither blocks nor log segments found. Creating new log.
03:14:49.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.048481 5690 log.cc:826] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0: Log is configured to *not* fsync() on all Append() calls
03:14:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.051544 5688 tablet_bootstrap.cc:492] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e: No bootstrap required, opened a new log
03:14:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.052065 5688 ts_tablet_manager.cc:1403] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e: Time spent bootstrapping tablet: real 0.023s user 0.009s sys 0.004s
03:14:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.052935 5690 tablet_bootstrap.cc:492] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0: No bootstrap required, opened a new log
03:14:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.053310 5690 ts_tablet_manager.cc:1403] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0: Time spent bootstrapping tablet: real 0.013s user 0.001s sys 0.010s
03:14:49.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.067919 5689 raft_consensus.cc:359] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.068823 5689 raft_consensus.cc:385] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:49.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.069123 5689 raft_consensus.cc:740] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823feaa369b047a89e2b9bdfaa6f3ca9, State: Initialized, Role: FOLLOWER
03:14:49.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.070114 5689 consensus_queue.cc:260] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.071879 5690 raft_consensus.cc:359] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } }
03:14:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.072822 5690 raft_consensus.cc:385] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.073118 5690 raft_consensus.cc:740] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27a6991a885b47e594fb08f14be5a0f0, State: Initialized, Role: FOLLOWER
03:14:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.073904 5690 consensus_queue.cc:260] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } }
03:14:49.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.076118 5689 ts_tablet_manager.cc:1434] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9: Time spent starting tablet: real 0.034s user 0.030s sys 0.004s
03:14:49.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.077049 5689 tablet_bootstrap.cc:492] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9: Bootstrap starting.
03:14:49.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.079360 5688 raft_consensus.cc:359] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.080229 5688 raft_consensus.cc:385] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:49.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.080518 5688 raft_consensus.cc:740] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5b6e1325a34b4f2c88e8626a3c208d6e, State: Initialized, Role: FOLLOWER
03:14:49.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.081467 5688 consensus_queue.cc:260] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.089082 5689 tablet_bootstrap.cc:654] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9: Neither blocks nor log segments found. Creating new log.
03:14:49.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.096140 5660 heartbeater.cc:499] Master 127.4.198.60:37911 was elected leader, sending a full tablet report...
03:14:49.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.096243 5688 ts_tablet_manager.cc:1434] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e: Time spent starting tablet: real 0.044s user 0.034s sys 0.004s
03:14:49.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.102873 5688 tablet_bootstrap.cc:492] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e: Bootstrap starting.
03:14:49.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.106904 5690 ts_tablet_manager.cc:1434] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0: Time spent starting tablet: real 0.053s user 0.031s sys 0.010s
03:14:49.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.108088 5690 tablet_bootstrap.cc:492] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0: Bootstrap starting.
03:14:49.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.108323 5515 tablet.cc:2379] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:14:49.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.110272 5689 tablet_bootstrap.cc:492] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9: No bootstrap required, opened a new log
03:14:49.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.110716 5689 ts_tablet_manager.cc:1403] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9: Time spent bootstrapping tablet: real 0.034s user 0.008s sys 0.005s
03:14:49.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.113229 5689 raft_consensus.cc:359] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } }
03:14:49.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.113868 5689 raft_consensus.cc:385] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:49.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.114147 5689 raft_consensus.cc:740] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823feaa369b047a89e2b9bdfaa6f3ca9, State: Initialized, Role: FOLLOWER
03:14:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.114260 5690 tablet_bootstrap.cc:654] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0: Neither blocks nor log segments found. Creating new log.
03:14:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.115396 5688 tablet_bootstrap.cc:654] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e: Neither blocks nor log segments found. Creating new log.
03:14:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.116398 5689 consensus_queue.cc:260] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } }
03:14:49.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.120546 5689 ts_tablet_manager.cc:1434] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9: Time spent starting tablet: real 0.010s user 0.003s sys 0.004s
03:14:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.140407 5690 tablet_bootstrap.cc:492] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0: No bootstrap required, opened a new log
03:14:49.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.140784 5690 ts_tablet_manager.cc:1403] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0: Time spent bootstrapping tablet: real 0.033s user 0.012s sys 0.015s
03:14:49.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.140897 5688 tablet_bootstrap.cc:492] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e: No bootstrap required, opened a new log
03:14:49.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.141201 5688 ts_tablet_manager.cc:1403] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e: Time spent bootstrapping tablet: real 0.040s user 0.013s sys 0.003s
03:14:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.142700 5690 raft_consensus.cc:359] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.143181 5690 raft_consensus.cc:385] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.142982 5688 raft_consensus.cc:359] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } }
03:14:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.143445 5688 raft_consensus.cc:385] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.143421 5690 raft_consensus.cc:740] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27a6991a885b47e594fb08f14be5a0f0, State: Initialized, Role: FOLLOWER
03:14:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.143668 5688 raft_consensus.cc:740] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5b6e1325a34b4f2c88e8626a3c208d6e, State: Initialized, Role: FOLLOWER
03:14:49.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.144029 5690 consensus_queue.cc:260] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.144301 5688 consensus_queue.cc:260] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } }
03:14:49.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.146625 5688 ts_tablet_manager.cc:1434] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e: Time spent starting tablet: real 0.005s user 0.000s sys 0.005s
03:14:49.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.154116 5690 ts_tablet_manager.cc:1434] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0: Time spent starting tablet: real 0.013s user 0.005s sys 0.000s
03:14:49.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.217968 5694 raft_consensus.cc:493] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:14:49.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.218431 5694 raft_consensus.cc:515] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.220749 5694 leader_election.cc:290] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5b6e1325a34b4f2c88e8626a3c208d6e (127.4.198.2:39509), 27a6991a885b47e594fb08f14be5a0f0 (127.4.198.3:42651)
03:14:49.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.230036 5696 raft_consensus.cc:493] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:14:49.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.230592 5696 raft_consensus.cc:515] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.232686 5695 raft_consensus.cc:493] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:14:49.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.233232 5695 raft_consensus.cc:515] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } }
03:14:49.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.234571 5696 leader_election.cc:290] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823feaa369b047a89e2b9bdfaa6f3ca9 (127.4.198.1:35799), 27a6991a885b47e594fb08f14be5a0f0 (127.4.198.3:42651)
03:14:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.237432 5695 leader_election.cc:290] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5b6e1325a34b4f2c88e8626a3c208d6e (127.4.198.2:39509), 823feaa369b047a89e2b9bdfaa6f3ca9 (127.4.198.1:35799)
03:14:49.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.238721 5615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8d77b3eaef1d44f8842ab9fd4ce8b99d" candidate_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "27a6991a885b47e594fb08f14be5a0f0" is_pre_election: true
03:14:49.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.239420 5615 raft_consensus.cc:2468] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 823feaa369b047a89e2b9bdfaa6f3ca9 in term 0.
03:14:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.241128 5466 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8d77b3eaef1d44f8842ab9fd4ce8b99d" candidate_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" is_pre_election: true
03:14:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.241869 5466 raft_consensus.cc:2468] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 823feaa369b047a89e2b9bdfaa6f3ca9 in term 0.
03:14:49.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.244060 5239 leader_election.cc:304] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5b6e1325a34b4f2c88e8626a3c208d6e, 823feaa369b047a89e2b9bdfaa6f3ca9; no voters:
03:14:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.245081 5694 raft_consensus.cc:2804] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:14:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.245421 5694 raft_consensus.cc:493] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:14:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.245709 5694 raft_consensus.cc:3060] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Advancing to term 1
03:14:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.247031 5466 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "66bcb4903dd74d08a0a8d147498e899b" candidate_uuid: "27a6991a885b47e594fb08f14be5a0f0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" is_pre_election: true
03:14:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.247567 5466 raft_consensus.cc:2468] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 27a6991a885b47e594fb08f14be5a0f0 in term 0.
03:14:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.248838 5537 leader_election.cc:304] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27a6991a885b47e594fb08f14be5a0f0, 5b6e1325a34b4f2c88e8626a3c208d6e; no voters:
03:14:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.249615 5695 raft_consensus.cc:2804] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:14:49.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.249989 5695 raft_consensus.cc:493] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:14:49.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.250314 5695 raft_consensus.cc:3060] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Advancing to term 1
03:14:49.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.254642 5694 raft_consensus.cc:515] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.254984 5317 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "66bcb4903dd74d08a0a8d147498e899b" candidate_uuid: "27a6991a885b47e594fb08f14be5a0f0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" is_pre_election: true
03:14:49.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.255805 5317 raft_consensus.cc:2468] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 27a6991a885b47e594fb08f14be5a0f0 in term 0.
03:14:49.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.257305 5466 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8d77b3eaef1d44f8842ab9fd4ce8b99d" candidate_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e"
03:14:49.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.258184 5466 raft_consensus.cc:3060] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Advancing to term 1
03:14:49.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.259127 5615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8d77b3eaef1d44f8842ab9fd4ce8b99d" candidate_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "27a6991a885b47e594fb08f14be5a0f0"
03:14:49.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.259639 5615 raft_consensus.cc:3060] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Advancing to term 1
03:14:49.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.261201 5614 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8d77b3eaef1d44f8842ab9fd4ce8b99d" candidate_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "27a6991a885b47e594fb08f14be5a0f0" is_pre_election: true
03:14:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.263545 5694 leader_election.cc:290] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [CANDIDATE]: Term 1 election: Requested vote from peers 5b6e1325a34b4f2c88e8626a3c208d6e (127.4.198.2:39509), 27a6991a885b47e594fb08f14be5a0f0 (127.4.198.3:42651)
03:14:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.265079 5695 raft_consensus.cc:515] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } }
03:14:49.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.265851 5317 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "8d77b3eaef1d44f8842ab9fd4ce8b99d" candidate_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" is_pre_election: true
03:14:49.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.265879 5615 raft_consensus.cc:2468] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 27a6991a885b47e594fb08f14be5a0f0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 823feaa369b047a89e2b9bdfaa6f3ca9 in term 1.
03:14:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.266530 5317 raft_consensus.cc:2393] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5b6e1325a34b4f2c88e8626a3c208d6e in current term 1: Already voted for candidate 823feaa369b047a89e2b9bdfaa6f3ca9 in this term.
03:14:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.266847 5695 leader_election.cc:290] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [CANDIDATE]: Term 1 election: Requested vote from peers 5b6e1325a34b4f2c88e8626a3c208d6e (127.4.198.2:39509), 823feaa369b047a89e2b9bdfaa6f3ca9 (127.4.198.1:35799)
03:14:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.267828 5238 leader_election.cc:304] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27a6991a885b47e594fb08f14be5a0f0, 823feaa369b047a89e2b9bdfaa6f3ca9; no voters:
03:14:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.268018 5465 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "66bcb4903dd74d08a0a8d147498e899b" candidate_uuid: "27a6991a885b47e594fb08f14be5a0f0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e"
03:14:49.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.268157 5388 leader_election.cc:304] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b6e1325a34b4f2c88e8626a3c208d6e; no voters: 27a6991a885b47e594fb08f14be5a0f0, 823feaa369b047a89e2b9bdfaa6f3ca9
03:14:49.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.268543 5694 raft_consensus.cc:2804] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 1 FOLLOWER]: Leader election won for term 1
03:14:49.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.268641 5465 raft_consensus.cc:3060] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Advancing to term 1
03:14:49.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.268983 5317 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "66bcb4903dd74d08a0a8d147498e899b" candidate_uuid: "27a6991a885b47e594fb08f14be5a0f0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9"
03:14:49.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.269495 5317 raft_consensus.cc:3060] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Advancing to term 1
03:14:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.268654 5466 raft_consensus.cc:2468] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 823feaa369b047a89e2b9bdfaa6f3ca9 in term 1.
03:14:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.271250 5696 raft_consensus.cc:2749] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 5b6e1325a34b4f2c88e8626a3c208d6e [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:14:49.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.275272 5465 raft_consensus.cc:2468] T 66bcb4903dd74d08a0a8d147498e899b P 5b6e1325a34b4f2c88e8626a3c208d6e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 27a6991a885b47e594fb08f14be5a0f0 in term 1.
03:14:49.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.275840 5694 raft_consensus.cc:697] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 1 LEADER]: Becoming Leader. State: Replica: 823feaa369b047a89e2b9bdfaa6f3ca9, State: Running, Role: LEADER
03:14:49.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.275987 5317 raft_consensus.cc:2468] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 27a6991a885b47e594fb08f14be5a0f0 in term 1.
03:14:49.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.276417 5537 leader_election.cc:304] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27a6991a885b47e594fb08f14be5a0f0, 5b6e1325a34b4f2c88e8626a3c208d6e; no voters:
03:14:49.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.276829 5694 consensus_queue.cc:237] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.277457 5695 raft_consensus.cc:2804] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [term 1 FOLLOWER]: Leader election won for term 1
03:14:49.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.279186 5695 raft_consensus.cc:697] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [term 1 LEADER]: Becoming Leader. State: Replica: 27a6991a885b47e594fb08f14be5a0f0, State: Running, Role: LEADER
03:14:49.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.280121 5695 consensus_queue.cc:237] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } }
03:14:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.294685 5108 catalog_manager.cc:5671] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 823feaa369b047a89e2b9bdfaa6f3ca9 (127.4.198.1). New cstate: current_term: 1 leader_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } health_report { overall_health: UNKNOWN } } }
03:14:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.295113 5107 catalog_manager.cc:5671] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 reported cstate change: term changed from 0 to 1, leader changed from <none> to 27a6991a885b47e594fb08f14be5a0f0 (127.4.198.3). New cstate: current_term: 1 leader_uuid: "27a6991a885b47e594fb08f14be5a0f0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } health_report { overall_health: UNKNOWN } } }
03:14:49.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.296638 5367 tablet.cc:2379] T 66bcb4903dd74d08a0a8d147498e899b P 823feaa369b047a89e2b9bdfaa6f3ca9: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:14:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.311123 5664 tablet.cc:2379] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.366104 5108 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54222:
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:14:49.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.368669 5108 catalog_manager.cc:7033] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:14:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.395110 5297 tablet_service.cc:1511] Processing CreateTablet for tablet 78974eeddc5f4534b2eeb0db74b16638 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=01ca269f9a4146ff9be8addb9a2cc99e]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:14:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.396814 5297 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 78974eeddc5f4534b2eeb0db74b16638. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.397092 5446 tablet_service.cc:1511] Processing CreateTablet for tablet 78974eeddc5f4534b2eeb0db74b16638 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=01ca269f9a4146ff9be8addb9a2cc99e]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:14:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.398197 5446 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 78974eeddc5f4534b2eeb0db74b16638. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:49.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.404013 5595 tablet_service.cc:1511] Processing CreateTablet for tablet 78974eeddc5f4534b2eeb0db74b16638 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=01ca269f9a4146ff9be8addb9a2cc99e]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:14:49.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.405606 5595 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 78974eeddc5f4534b2eeb0db74b16638. 1 dirs total, 0 dirs full, 0 dirs failed
03:14:49.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.410877 5689 tablet_bootstrap.cc:492] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: Bootstrap starting.
03:14:49.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.410913 5688 tablet_bootstrap.cc:492] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e: Bootstrap starting.
03:14:49.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.416224 5689 tablet_bootstrap.cc:654] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: Neither blocks nor log segments found. Creating new log.
03:14:49.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.417114 5688 tablet_bootstrap.cc:654] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e: Neither blocks nor log segments found. Creating new log.
03:14:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.421057 5690 tablet_bootstrap.cc:492] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0: Bootstrap starting.
03:14:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.425274 5688 tablet_bootstrap.cc:492] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e: No bootstrap required, opened a new log
03:14:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.425695 5688 ts_tablet_manager.cc:1403] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e: Time spent bootstrapping tablet: real 0.015s user 0.001s sys 0.010s
03:14:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.425791 5689 tablet_bootstrap.cc:492] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: No bootstrap required, opened a new log
03:14:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.426133 5689 ts_tablet_manager.cc:1403] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.010s
03:14:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.427446 5690 tablet_bootstrap.cc:654] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0: Neither blocks nor log segments found. Creating new log.
03:14:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.427975 5689 raft_consensus.cc:359] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.428193 5688 raft_consensus.cc:359] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.428643 5689 raft_consensus.cc:385] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.428826 5688 raft_consensus.cc:385] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.428922 5689 raft_consensus.cc:740] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823feaa369b047a89e2b9bdfaa6f3ca9, State: Initialized, Role: FOLLOWER
03:14:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.429104 5688 raft_consensus.cc:740] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5b6e1325a34b4f2c88e8626a3c208d6e, State: Initialized, Role: FOLLOWER
03:14:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.429632 5689 consensus_queue.cc:260] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.429811 5688 consensus_queue.cc:260] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.430976 5694 tablet_replica.cc:442] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } }
03:14:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.431012 5696 tablet_replica.cc:442] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } }
03:14:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.431631 5694 tablet_replica.cc:445] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: This TxnStatusTablet replica's current role is: FOLLOWER
03:14:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.431643 5696 tablet_replica.cc:445] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e: This TxnStatusTablet replica's current role is: FOLLOWER
03:14:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.434305 5689 ts_tablet_manager.cc:1434] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: Time spent starting tablet: real 0.008s user 0.006s sys 0.001s
03:14:49.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.436236 5688 ts_tablet_manager.cc:1434] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e: Time spent starting tablet: real 0.010s user 0.007s sys 0.000s
03:14:49.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.451046 5694 raft_consensus.cc:493] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:14:49.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.451471 5694 raft_consensus.cc:515] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.452227 5690 tablet_bootstrap.cc:492] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0: No bootstrap required, opened a new log
03:14:49.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.452594 5690 ts_tablet_manager.cc:1403] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0: Time spent bootstrapping tablet: real 0.032s user 0.006s sys 0.005s
03:14:49.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.453306 5694 leader_election.cc:290] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5b6e1325a34b4f2c88e8626a3c208d6e (127.4.198.2:39509), 27a6991a885b47e594fb08f14be5a0f0 (127.4.198.3:42651)
03:14:49.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.454000 5465 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "78974eeddc5f4534b2eeb0db74b16638" candidate_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" is_pre_election: true
03:14:49.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.454558 5465 raft_consensus.cc:2468] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 823feaa369b047a89e2b9bdfaa6f3ca9 in term 0.
03:14:49.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.454562 5615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "78974eeddc5f4534b2eeb0db74b16638" candidate_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "27a6991a885b47e594fb08f14be5a0f0" is_pre_election: true
03:14:49.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.454988 5690 raft_consensus.cc:359] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.455626 5690 raft_consensus.cc:385] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:14:49.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.455494 5239 leader_election.cc:304] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5b6e1325a34b4f2c88e8626a3c208d6e, 823feaa369b047a89e2b9bdfaa6f3ca9; no voters:
03:14:49.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.455828 5690 raft_consensus.cc:740] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27a6991a885b47e594fb08f14be5a0f0, State: Initialized, Role: FOLLOWER
03:14:49.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.456123 5694 raft_consensus.cc:2804] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:14:49.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.456413 5694 raft_consensus.cc:493] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:14:49.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.456677 5694 raft_consensus.cc:3060] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 0 FOLLOWER]: Advancing to term 1
03:14:49.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.456446 5690 consensus_queue.cc:260] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.457243 5615 raft_consensus.cc:2468] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 823feaa369b047a89e2b9bdfaa6f3ca9 in term 0.
03:14:49.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.457716 5695 tablet_replica.cc:442] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } }
03:14:49.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.458483 5695 tablet_replica.cc:445] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0: This TxnStatusTablet replica's current role is: FOLLOWER
03:14:49.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.458606 5690 ts_tablet_manager.cc:1434] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0: Time spent starting tablet: real 0.006s user 0.004s sys 0.000s
03:14:49.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.463208 5694 raft_consensus.cc:515] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.464895 5694 leader_election.cc:290] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [CANDIDATE]: Term 1 election: Requested vote from peers 5b6e1325a34b4f2c88e8626a3c208d6e (127.4.198.2:39509), 27a6991a885b47e594fb08f14be5a0f0 (127.4.198.3:42651)
03:14:49.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.465960 5615 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "78974eeddc5f4534b2eeb0db74b16638" candidate_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "27a6991a885b47e594fb08f14be5a0f0"
03:14:49.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.466464 5615 raft_consensus.cc:3060] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0 [term 0 FOLLOWER]: Advancing to term 1
03:14:49.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.466504 5465 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "78974eeddc5f4534b2eeb0db74b16638" candidate_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e"
03:14:49.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.466907 5465 raft_consensus.cc:3060] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e [term 0 FOLLOWER]: Advancing to term 1
03:14:49.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.471469 5465 raft_consensus.cc:2468] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 823feaa369b047a89e2b9bdfaa6f3ca9 in term 1.
03:14:49.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.471504 5615 raft_consensus.cc:2468] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 823feaa369b047a89e2b9bdfaa6f3ca9 in term 1.
03:14:49.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.472522 5239 leader_election.cc:304] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5b6e1325a34b4f2c88e8626a3c208d6e, 823feaa369b047a89e2b9bdfaa6f3ca9; no voters:
03:14:49.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.473484 5694 raft_consensus.cc:2804] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 1 FOLLOWER]: Leader election won for term 1
03:14:49.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.473800 5694 raft_consensus.cc:697] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [term 1 LEADER]: Becoming Leader. State: Replica: 823feaa369b047a89e2b9bdfaa6f3ca9, State: Running, Role: LEADER
03:14:49.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.474402 5694 consensus_queue.cc:237] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } }
03:14:49.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.479795 5706 tablet_replica.cc:442] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: TxnStatusTablet state changed. Reason: New leader 823feaa369b047a89e2b9bdfaa6f3ca9. Latest consensus state: current_term: 1 leader_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } }
03:14:49.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.480528 5706 tablet_replica.cc:445] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: This TxnStatusTablet replica's current role is: LEADER
03:14:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.486045 5108 catalog_manager.cc:5671] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 reported cstate change: term changed from 0 to 1, leader changed from <none> to 823feaa369b047a89e2b9bdfaa6f3ca9 (127.4.198.1). New cstate: current_term: 1 leader_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } health_report { overall_health: UNKNOWN } } }
03:14:49.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.491473 5723 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
03:14:49.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.492051 5723 txn_status_manager.cc:930] Loading transaction status metadata into memory...
03:14:49.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.538111 5615 raft_consensus.cc:1275] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0 [term 1 FOLLOWER]: Refusing update from remote peer 823feaa369b047a89e2b9bdfaa6f3ca9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:14:49.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.539290 5465 raft_consensus.cc:1275] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e [term 1 FOLLOWER]: Refusing update from remote peer 823feaa369b047a89e2b9bdfaa6f3ca9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:14:49.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.539620 5706 consensus_queue.cc:1048] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:14:49.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.541553 5706 consensus_queue.cc:1048] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:14:49.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.574656 5696 tablet_replica.cc:442] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e: TxnStatusTablet state changed. Reason: New leader 823feaa369b047a89e2b9bdfaa6f3ca9. Latest consensus state: current_term: 1 leader_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } }
03:14:49.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.574899 5695 tablet_replica.cc:442] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0: TxnStatusTablet state changed. Reason: New leader 823feaa369b047a89e2b9bdfaa6f3ca9. Latest consensus state: current_term: 1 leader_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } }
03:14:49.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.575333 5696 tablet_replica.cc:445] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e: This TxnStatusTablet replica's current role is: FOLLOWER
03:14:49.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.575512 5695 tablet_replica.cc:445] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0: This TxnStatusTablet replica's current role is: FOLLOWER
03:14:49.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.591354 5726 mvcc.cc:204] Tried to move back new op lower bound from 7265499912332656640 to 7265499912092618752. Current Snapshot: MvccSnapshot[applied={T|T < 7265499912332656640}]
03:14:49.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.602772 5694 tablet_replica.cc:442] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } }
03:14:49.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.603421 5694 tablet_replica.cc:445] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: This TxnStatusTablet replica's current role is: LEADER
03:14:49.616 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
03:14:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.615747 5696 tablet_replica.cc:442] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } }
03:14:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.616415 5696 tablet_replica.cc:445] T 78974eeddc5f4534b2eeb0db74b16638 P 5b6e1325a34b4f2c88e8626a3c208d6e: This TxnStatusTablet replica's current role is: FOLLOWER
03:14:49.619 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 38d0a76607c74e47b30cf514a7631d2b
03:14:49.619 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.198.60:37911
03:14:49.620 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:14:49.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.620482 4888 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu with pid 5065
03:14:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.631735 5728 mvcc.cc:204] Tried to move back new op lower bound from 7265499912332656640 to 7265499912092618752. Current Snapshot: MvccSnapshot[applied={T|T < 7265499912332656640}]
03:14:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.633066 5727 mvcc.cc:204] Tried to move back new op lower bound from 7265499912332656640 to 7265499912092618752. Current Snapshot: MvccSnapshot[applied={T|T < 7265499912332656640}]
03:14:49.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.646602 5695 tablet_replica.cc:442] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } }
03:14:49.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.647250 5695 tablet_replica.cc:445] T 78974eeddc5f4534b2eeb0db74b16638 P 27a6991a885b47e594fb08f14be5a0f0: This TxnStatusTablet replica's current role is: FOLLOWER
03:14:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.671262 5239 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.4.198.2:39509, user_credentials={real_user=slave}} blocked reactor thread for 53673.1us
03:14:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.672623 5733 tablet_replica.cc:442] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 } } peers { permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 } } peers { permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 } } }
03:14:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.673411 5733 tablet_replica.cc:445] T 78974eeddc5f4534b2eeb0db74b16638 P 823feaa369b047a89e2b9bdfaa6f3ca9: This TxnStatusTablet replica's current role is: LEADER
03:14:49.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.679558 5706 consensus_queue.cc:1048] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:14:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.687301 5695 consensus_queue.cc:1048] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" member_type: VOTER last_known_addr { host: "127.4.198.2" port: 39509 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:14:49.718 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.198.60:37911(127.4.198.60:37911)] handling channelInactive
03:14:49.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.198.60:37911(127.4.198.60:37911)] cleaning up while in state READY due to: connection closed
03:14:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.719755 5240 connection.cc:537] client connection to 127.4.198.60:37911 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:14:49.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.720065 5538 connection.cc:537] client connection to 127.4.198.60:37911 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:14:49.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.720618 5363 heartbeater.cc:646] Failed to heartbeat to 127.4.198.60:37911 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:14:49.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.720938 5389 connection.cc:537] client connection to 127.4.198.60:37911 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:14:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.722379 5660 heartbeater.cc:646] Failed to heartbeat to 127.4.198.60:37911 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:14:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.723340 5511 heartbeater.cc:646] Failed to heartbeat to 127.4.198.60:37911 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:14:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.722122 5738 consensus_queue.cc:1048] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" member_type: VOTER last_known_addr { host: "127.4.198.1" port: 35799 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:14:49.736 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:14:49.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.807868 5694 consensus_queue.cc:1048] T 8d77b3eaef1d44f8842ab9fd4ce8b99d P 823feaa369b047a89e2b9bdfaa6f3ca9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" member_type: VOTER last_known_addr { host: "127.4.198.3" port: 42651 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:14:49.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.915681 5743 raft_consensus.cc:493] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader cfc16134dcdc440a843537d928cf2c3e)
03:14:49.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.916465 5743 raft_consensus.cc:515] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:49.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.925339 5743 leader_election.cc:290] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 79bba6a868c54f17a187e7418fe73c49 (127.4.198.62:43377), cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911)
03:14:49.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.926033 5744 raft_consensus.cc:493] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader cfc16134dcdc440a843537d928cf2c3e)
03:14:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.927461 4940 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be6186a03e5f46e685128bf5ce84b5f4" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "79bba6a868c54f17a187e7418fe73c49" is_pre_election: true
03:14:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.929275 5004 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111) [suppressed 6 similar messages]
03:14:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.929551 5744 raft_consensus.cc:515] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.930691 4940 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate be6186a03e5f46e685128bf5ce84b5f4 in term 1.
03:14:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.934194 5002 leader_election.cc:304] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 79bba6a868c54f17a187e7418fe73c49, be6186a03e5f46e685128bf5ce84b5f4; no voters:
03:14:49.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.935328 5743 raft_consensus.cc:2804] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 1 FOLLOWER]: Leader pre-election won for term 2
03:14:49.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.935727 5743 raft_consensus.cc:493] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 1 FOLLOWER]: Starting leader election (detected failure of leader cfc16134dcdc440a843537d928cf2c3e)
03:14:49.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.936187 5743 raft_consensus.cc:3060] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 1 FOLLOWER]: Advancing to term 2
03:14:49.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.942953 5744 leader_election.cc:290] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers be6186a03e5f46e685128bf5ce84b5f4 (127.4.198.61:45443), cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911)
03:14:49.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.944995 5028 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "79bba6a868c54f17a187e7418fe73c49" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "be6186a03e5f46e685128bf5ce84b5f4" is_pre_election: true
03:14:49.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.945372 5743 raft_consensus.cc:515] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:49.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.946724 5028 raft_consensus.cc:2393] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 79bba6a868c54f17a187e7418fe73c49 in current term 2: Already voted for candidate be6186a03e5f46e685128bf5ce84b5f4 in this term.
03:14:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.948251 4940 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "be6186a03e5f46e685128bf5ce84b5f4" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "79bba6a868c54f17a187e7418fe73c49"
03:14:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.948825 4940 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 1 FOLLOWER]: Advancing to term 2
03:14:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.949831 5004 leader_election.cc:336] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911): Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.952368 5743 leader_election.cc:290] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [CANDIDATE]: Term 2 election: Requested vote from peers 79bba6a868c54f17a187e7418fe73c49 (127.4.198.62:43377), cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911)
03:14:49.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.956702 4940 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate be6186a03e5f46e685128bf5ce84b5f4 in term 2.
03:14:49.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.957712 5004 leader_election.cc:336] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911): Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:49.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.958288 5002 leader_election.cc:304] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 79bba6a868c54f17a187e7418fe73c49, be6186a03e5f46e685128bf5ce84b5f4; no voters: cfc16134dcdc440a843537d928cf2c3e
03:14:49.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.958530 4919 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111) [suppressed 13 similar messages]
03:14:49.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.959069 5743 raft_consensus.cc:2804] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 2 FOLLOWER]: Leader election won for term 2
03:14:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.960980 5743 raft_consensus.cc:697] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [term 2 LEADER]: Becoming Leader. State: Replica: be6186a03e5f46e685128bf5ce84b5f4, State: Running, Role: LEADER
03:14:49.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.961879 5743 consensus_queue.cc:237] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:49.962489 4919 leader_election.cc:336] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911): Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111)
03:14:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.962906 4919 leader_election.cc:304] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 79bba6a868c54f17a187e7418fe73c49; no voters: be6186a03e5f46e685128bf5ce84b5f4, cfc16134dcdc440a843537d928cf2c3e
03:14:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.963598 5744 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
03:14:49.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.965199 5749 sys_catalog.cc:455] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader be6186a03e5f46e685128bf5ce84b5f4. Latest consensus state: current_term: 2 leader_uuid: "be6186a03e5f46e685128bf5ce84b5f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.965912 5749 sys_catalog.cc:458] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [sys.catalog]: This master's current role is: LEADER
03:14:49.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.967947 5750 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:14:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.977314 5750 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=01ca269f9a4146ff9be8addb9a2cc99e]
03:14:49.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.979107 5750 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b]
03:14:49.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.985711 5750 tablet_loader.cc:96] loaded metadata for tablet 66bcb4903dd74d08a0a8d147498e899b (table txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b])
03:14:49.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.987066 5750 tablet_loader.cc:96] loaded metadata for tablet 78974eeddc5f4534b2eeb0db74b16638 (table kudu_system.kudu_transactions [id=01ca269f9a4146ff9be8addb9a2cc99e])
03:14:49.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.988437 5750 tablet_loader.cc:96] loaded metadata for tablet 8d77b3eaef1d44f8842ab9fd4ce8b99d (table txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b])
03:14:49.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.989668 5750 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:14:49.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.993846 5750 catalog_manager.cc:1269] Loaded cluster ID: 5aff527ef8e1466fb0288a917ecfa345
03:14:49.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.994122 5750 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:14:49.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:49.998783 5750 catalog_manager.cc:1514] Loading token signing keys...
03:14:50.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:50.002852 5750 catalog_manager.cc:6055] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4: Loaded TSK: 0
03:14:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:50.004155 5750 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:14:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:50.060659 5004 consensus_peers.cc:597] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 -> Peer cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911): Couldn't send request to peer cfc16134dcdc440a843537d928cf2c3e. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:14:50.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:50.087260 4940 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 2 FOLLOWER]: Refusing update from remote peer be6186a03e5f46e685128bf5ce84b5f4: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
03:14:50.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:50.088768 5749 consensus_queue.cc:1048] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
03:14:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:50.091838 5744 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: SysCatalogTable state changed. Reason: New leader be6186a03e5f46e685128bf5ce84b5f4. Latest consensus state: current_term: 2 leader_uuid: "be6186a03e5f46e685128bf5ce84b5f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:50.092563 5744 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: This master's current role is: FOLLOWER
03:14:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:50.095840 5743 sys_catalog.cc:455] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "be6186a03e5f46e685128bf5ce84b5f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:50.096540 5743 sys_catalog.cc:458] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 [sys.catalog]: This master's current role is: LEADER
03:14:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:50.098392 5744 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "be6186a03e5f46e685128bf5ce84b5f4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:50.098943 5744 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: This master's current role is: FOLLOWER
03:14:50.131 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.60:37911(127.4.198.60:37911)] connecting to peer
03:14:50.134 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.4.198.60:37911(127.4.198.60:37911): Connection refused: /127.4.198.60:37911
03:14:50.134 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.4.198.60:37911(127.4.198.60:37911)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.60:37911(127.4.198.60:37911): Connection refused: /127.4.198.60:37911
03:14:50.134 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location master-127.4.198.60:37911(127.4.198.60:37911) for tablet Kudu Master: Failed to connect to peer master-127.4.198.60:37911(127.4.198.60:37911): Connection refused: /127.4.198.60:37911
03:14:50.152 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.60:37911(127.4.198.60:37911)] connecting to peer
03:14:50.154 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.4.198.60:37911(127.4.198.60:37911): Connection refused: /127.4.198.60:37911
03:14:50.154 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.4.198.60:37911(127.4.198.60:37911)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.60:37911(127.4.198.60:37911): Connection refused: /127.4.198.60:37911
03:14:50.154 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.60:37911: Failed to connect to peer master-127.4.198.60:37911(127.4.198.60:37911): Connection refused: /127.4.198.60:37911
03:14:50.158 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:14:50.158 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=27), Traces: [0ms] sending RPC to server master-127.4.198.60:37911, [3ms] received response from server master-127.4.198.60:37911: Network error: Failed to connect to peer master-127.4.198.60:37911(127.4.198.60:37911): Connection refused: /127.4.198.60:37911, [3ms] delaying RPC due to: Network error: Failed to connect to peer master-127.4.198.60:37911(127.4.198.60:37911): Connection refused: /127.4.198.60:37911, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [23ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: Network error: Failed to connect to peer master-127.4.198.60:37911(127.4.198.60:37911): Connection refused: /127.4.198.60:37911, [25ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [26ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: OK, deferred=Deferred@1067603588(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$238/0x00007f4b84336a70@565e1205, errback=org.apache.kudu.client.KuduTransaction$$Lambda$239/0x00007f4b84336c98@6e492664)) after lookup
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:50.215575 5018 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53438:
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:14:50.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:50.219646 5018 catalog_manager.cc:7033] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:14:50.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:50.609264 5004 consensus_peers.cc:597] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 -> Peer cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911): Couldn't send request to peer cfc16134dcdc440a843537d928cf2c3e. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:14:51.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:51.144163 5004 consensus_peers.cc:597] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 -> Peer cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911): Couldn't send request to peer cfc16134dcdc440a843537d928cf2c3e. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
03:14:51.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:51.175457 5661 heartbeater.cc:499] Master 127.4.198.61:45443 was elected leader, sending a full tablet report...
03:14:51.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:51.311353 5364 heartbeater.cc:499] Master 127.4.198.61:45443 was elected leader, sending a full tablet report...
03:14:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:51.658865 5004 consensus_peers.cc:597] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 -> Peer cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911): Couldn't send request to peer cfc16134dcdc440a843537d928cf2c3e. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
03:14:51.744 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 66bcb4903dd74d08a0a8d147498e899b for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
03:14:51.744 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet 8d77b3eaef1d44f8842ab9fd4ce8b99d for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
03:14:51.744 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=66bcb4903dd74d08a0a8d147498e899b@[27a6991a885b47e594fb08f14be5a0f0(127.4.198.3:42651)[L],5b6e1325a34b4f2c88e8626a3c208d6e(127.4.198.2:39509),823feaa369b047a89e2b9bdfaa6f3ca9(127.4.198.1:35799)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=8d77b3eaef1d44f8842ab9fd4ce8b99d@[27a6991a885b47e594fb08f14be5a0f0(127.4.198.3:42651),5b6e1325a34b4f2c88e8626a3c208d6e(127.4.198.2:39509),823feaa369b047a89e2b9bdfaa6f3ca9(127.4.198.1:35799)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
03:14:51.744 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.198.61:45443, [3ms] Sub RPC GetTableLocations: received response from server master-127.4.198.61:45443: OK, deferred=null) after lookup
03:14:51.744 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 27a6991a885b47e594fb08f14be5a0f0(127.4.198.3:42651)] connecting to peer
03:14:51.746 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 27a6991a885b47e594fb08f14be5a0f0(127.4.198.3:42651)] Successfully connected to peer
03:14:51.759 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0x54634b60, L:/127.0.0.1:50658 - R:/127.4.198.3:42651] using TOKEN/null
03:14:51.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:51.765760 5559 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.4.198.60:37911)
03:14:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:51.824714 5512 heartbeater.cc:499] Master 127.4.198.61:45443 was elected leader, sending a full tablet report...
03:14:51.872 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.198.60:37911
03:14:51.872 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:14:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:51.874078 4888 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu
03:14:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal
03:14:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/logs
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data/info.pb
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.60:37911
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.60
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37305
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46207/jwks.json
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:14:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:14:52.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.141156 5004 consensus_peers.cc:597] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 -> Peer cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911): Couldn't send request to peer cfc16134dcdc440a843537d928cf2c3e. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
03:14:52.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.167950 5798 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:14:52.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.168509 5798 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:14:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.168987 5798 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:14:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.202116 5798 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:14:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.202387 5798 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:14:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.202548 5798 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:14:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.202737 5798 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:14:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.202936 5798 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:14:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.203122 5798 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:14:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.203264 5798 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46207/jwks.json
03:14:52.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:52.240598 5798 master_runner.cc:386] Master server non-default flags:
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.198.20:40713
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.198.62:43377,127.4.198.61:45443,127.4.198.60:37911
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.198.60:37911
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46207/jwks.json
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data/info.pb
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.198.60
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37305
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/logs
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 07148adc8b1ea21ab089857e39de46f7fa68ddab
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Mar 2026 02:43:20 UTC on e7f111948823
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10983
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:14:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:52.241809 5798 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:14:52.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:52.243304 5798 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:14:52.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.253363 5806 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:52.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.667713 5004 consensus_peers.cc:597] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 -> Peer cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911): Couldn't send request to peer cfc16134dcdc440a843537d928cf2c3e. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
03:14:53.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:53.229497 5004 consensus_peers.cc:597] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 -> Peer cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911): Couldn't send request to peer cfc16134dcdc440a843537d928cf2c3e. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
03:14:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:53.656589 5805 debug-util.cc:398] Leaking SignalData structure 0x7b08000376a0 after lost signal to thread 5798
03:14:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:53.666813 5805 kernel_stack_watchdog.cc:198] Thread 5798 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
03:14:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:14:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:14:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:14:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:14:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:52.254320 5807 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:53.667505 5798 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.413s user 0.519s sys 0.687s
03:14:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:53.667858 5798 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.414s user 0.519s sys 0.687s
03:14:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:53.669101 5809 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:14:53.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:53.671320 5808 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1412 milliseconds
03:14:53.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.671367 5798 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:14:53.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.672665 5798 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:14:53.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.674984 5798 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:14:53.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.676312 5798 hybrid_clock.cc:648] HybridClock initialized: now 1773803693676268 us; error 41 us; skew 500 ppm
03:14:53.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.677110 5798 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:14:53.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.682880 5798 webserver.cc:492] Webserver started at http://127.4.198.60:37305/ using document root <none> and password file <none>
03:14:53.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.683815 5798 fs_manager.cc:362] Metadata directory not provided
03:14:53.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.684051 5798 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:14:53.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:53.690001 5004 consensus_peers.cc:597] T 00000000000000000000000000000000 P be6186a03e5f46e685128bf5ce84b5f4 -> Peer cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911): Couldn't send request to peer cfc16134dcdc440a843537d928cf2c3e. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.60:37911: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
03:14:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.692351 5798 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
03:14:53.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.697172 5821 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:14:53.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.698344 5798 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:14:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.698724 5798 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data,/tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal
03:14:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cfc16134dcdc440a843537d928cf2c3e"
03:14:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-18 03:14:40 on dist-test-slave-vs5b"
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.700692 5798 fs_report.cc:389] FS layout report
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/wal
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data/data
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:14:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:14:53.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.746605 5798 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:14:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.748021 5798 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:14:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.748462 5798 kserver.cc:163] Server-wide thread pool size limit: 3276
03:14:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.839295 5798 rpc_server.cc:307] RPC server started. Bound to: 127.4.198.60:37911
03:14:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.839438 5882 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.198.60:37911 every 8 connection(s)
03:14:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.842042 5798 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskaVXZmI/test-tmp/mini-kudu-cluster6640227268771980876/master-2/data/info.pb
03:14:53.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.843709 4888 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu as pid 5798
03:14:53.844 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:14:53.844 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.198.61:45443
03:14:53.844 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:14:53.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.845733 4888 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu with pid 4980
03:14:53.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.854614 5883 sys_catalog.cc:263] Verifying existing consensus state
03:14:53.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.859652 5883 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: Bootstrap starting.
03:14:53.873 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.4.198.61:45443(127.4.198.61:45443)] handling channelInactive
03:14:53.873 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state READY due to: connection closed
03:14:53.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:53.873963 4920 connection.cc:537] server connection from 127.0.0.1:42220 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:14:53.874 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:14:53.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:53.874851 5387 connection.cc:537] client connection to 127.4.198.61:45443 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:14:53.892 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:53.895 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.895 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.895 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location master-127.4.198.61:45443(127.4.198.61:45443) for tablet Kudu Master: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.903843 5883 log.cc:826] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: Log is configured to *not* fsync() on all Append() calls
03:14:53.912 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.60:37911(127.4.198.60:37911)] connecting to peer
03:14:53.913 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:53.914 [DEBUG - kudu-nio-7] (Connection.java:757) [peer master-127.4.198.60:37911(127.4.198.60:37911)] Successfully connected to peer
03:14:53.914 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.914 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.914 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.914502 5511 heartbeater.cc:344] Connected to a master server at 127.4.198.60:37911
03:14:53.917 [DEBUG - kudu-nio-7] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.4.198.60
03:14:53.932 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x45e41702, L:/127.0.0.1:55034 - R:/127.4.198.60:37911] using SASL/PLAIN
03:14:53.934 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.60:37911: Catalog manager is not initialized. State: Starting
03:14:53.935 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:53.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.942629 5883 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:14:53.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.943482 5883 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: Bootstrap complete.
03:14:53.952 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:53.953 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.953 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.953 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.955 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.60:37911: Catalog manager is not initialized. State: Starting
03:14:53.955 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:53.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.963786 5883 raft_consensus.cc:359] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:53.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.966065 5883 raft_consensus.cc:740] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: cfc16134dcdc440a843537d928cf2c3e, State: Initialized, Role: FOLLOWER
03:14:53.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.966879 5883 consensus_queue.cc:260] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.969700 5890 sys_catalog.cc:455] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.970479 5890 sys_catalog.cc:458] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: This master's current role is: FOLLOWER
03:14:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:53.971484 5883 sys_catalog.cc:565] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: configured and running, proceeding with master startup.
03:14:53.972 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:53.974 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.974 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.974 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.974 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.60:37911: Catalog manager is not initialized. State: Starting
03:14:53.976 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:53.992 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:53.993 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.993 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.994 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:53.999 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.015691 5901 catalog_manager.cc:1269] Loaded cluster ID: 5aff527ef8e1466fb0288a917ecfa345
03:14:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.016216 5901 catalog_manager.cc:1562] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: loading cluster ID for follower catalog manager: success
03:14:54.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.024938 5901 catalog_manager.cc:1584] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: acquiring CA information for follower catalog manager: success
03:14:54.032 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.030344 5901 catalog_manager.cc:1612] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:14:54.033 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.033 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.033 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.035 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.028527 5798 master_runner.cc:186] Error getting master registration for 127.4.198.61:45443: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111),
03:14:54.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.047873 5798 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:14:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.086920 4974 debug-util.cc:398] Leaking SignalData structure 0x7b08000a5980 after lost signal to thread 4901
03:14:54.092 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.096 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.096 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.096 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.096 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.116 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.117 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.117 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.117 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.120 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.121 [DEBUG - kudu-nio-7] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=228), Traces: [0ms] sending RPC to server master-127.4.198.61:45443, [3ms] received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [3ms] delaying RPC due to: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [22ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [25ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [43ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [61ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [84ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [102ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [103ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [106ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [107ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [141ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [142ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [202ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [203ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [204ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [204ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [223ms] refreshing cache from master, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [225ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [228ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [228ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [229ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, deferred=Deferred@616223626(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$238/0x00007f4b84336a70@4ab5669d -> (continuation of Deferred@1967832055 after retry RPC after error@1116052102) -> (continuation of Deferred@810629087 after retry RPC after error@1370566342) -> (continuation of Deferred@1896129028 after retry RPC after error@523978504) -> (continuation of Deferred@653096191 after retry RPC after error@2121447930) -> (continuation of Deferred@716320946 after retry RPC after error@1878274539) -> (continuation of Deferred@578026086 after retry RPC after error@1235565243), errback=org.apache.kudu.client.KuduTransaction$$Lambda$239/0x00007f4b84336c98@7229b49d -> (continuation of Deferred@1967832055 after retry RPC after error@1116052102) -> (continuation of Deferred@810629087 after retry RPC after error@1370566342) -> (continuation of Deferred@1896129028 after retry RPC after error@523978504) -> (continuation of Deferred@653096191 after retry RPC after error@2121447930) -> (continuation of Deferred@716320946 after retry RPC after error@1878274539) -> (continuation of Deferred@578026086 after retry RPC after error@1235565243)))
03:14:54.122 [DEBUG - kudu-nio-7] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=228), Traces: [0ms] sending RPC to server master-127.4.198.61:45443, [3ms] received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [3ms] delaying RPC due to: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [22ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [25ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [43ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [61ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [84ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [102ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [103ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [106ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [107ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [141ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [142ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [202ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [203ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [204ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [204ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [223ms] refreshing cache from master, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [225ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [228ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [228ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [229ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, deferred=Deferred@616223626(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$238/0x00007f4b84336a70@4ab5669d -> (continuation of Deferred@1967832055 after retry RPC after error@1116052102) -> (continuation of Deferred@810629087 after retry RPC after error@1370566342) -> (continuation of Deferred@1896129028 after retry RPC after error@523978504) -> (continuation of Deferred@653096191 after retry RPC after error@2121447930) -> (continuation of Deferred@716320946 after retry RPC after error@1878274539) -> (continuation of Deferred@578026086 after retry RPC after error@1235565243), errback=org.apache.kudu.client.KuduTransaction$$Lambda$239/0x00007f4b84336c98@7229b49d -> (continuation of Deferred@1967832055 after retry RPC after error@1116052102) -> (continuation of Deferred@810629087 after retry RPC after error@1370566342) -> (continuation of Deferred@1896129028 after retry RPC after error@523978504) -> (continuation of Deferred@653096191 after retry RPC after error@2121447930) -> (continuation of Deferred@716320946 after retry RPC after error@1878274539) -> (continuation of Deferred@578026086 after retry RPC after error@1235565243)))
03:14:54.133 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.135 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.135 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.135 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.138 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.152 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.156 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.156 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.157 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.157 [WARN - pool-40-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.169459 5905 raft_consensus.cc:493] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader be6186a03e5f46e685128bf5ce84b5f4)
03:14:54.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.170323 5905 raft_consensus.cc:515] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:54.172 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.173 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.173 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.173 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.178 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.192 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.193 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.193 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.193 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.224428 5905 leader_election.cc:290] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers be6186a03e5f46e685128bf5ce84b5f4 (127.4.198.61:45443), cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911)
03:14:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.235677 5848 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "79bba6a868c54f17a187e7418fe73c49" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "cfc16134dcdc440a843537d928cf2c3e" is_pre_election: true
03:14:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.236711 5848 raft_consensus.cc:2468] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 79bba6a868c54f17a187e7418fe73c49 in term 1.
03:14:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.238278 4919 leader_election.cc:304] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 79bba6a868c54f17a187e7418fe73c49, cfc16134dcdc440a843537d928cf2c3e; no voters:
03:14:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.239233 5905 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 2 FOLLOWER]: Leader pre-election won for term 3
03:14:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.239656 5905 raft_consensus.cc:493] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 2 FOLLOWER]: Starting leader election (detected failure of leader be6186a03e5f46e685128bf5ce84b5f4)
03:14:54.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.240130 5905 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 2 FOLLOWER]: Advancing to term 3
03:14:54.246 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.247221 5660 heartbeater.cc:344] Connected to a master server at 127.4.198.60:37911
03:14:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.250798 4917 leader_election.cc:336] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer be6186a03e5f46e685128bf5ce84b5f4 (127.4.198.61:45443): Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111)
03:14:54.252 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.253 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.253 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.253 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.255456 5838 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "27a6991a885b47e594fb08f14be5a0f0" instance_seqno: 1773803688764098) as {username='slave'} at 127.4.198.3:33151; Asking this server to re-register.
03:14:54.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.257004 5660 heartbeater.cc:461] Registering TS with master...
03:14:54.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.260251 5905 raft_consensus.cc:515] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:54.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.261112 5838 ts_manager.cc:194] Registered new tserver with Master: 27a6991a885b47e594fb08f14be5a0f0 (127.4.198.3:42651)
03:14:54.266 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.273428 4917 leader_election.cc:336] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer be6186a03e5f46e685128bf5ce84b5f4 (127.4.198.61:45443): Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111)
03:14:54.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.275513 5848 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "79bba6a868c54f17a187e7418fe73c49" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "cfc16134dcdc440a843537d928cf2c3e"
03:14:54.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.276063 5848 raft_consensus.cc:3060] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 1 FOLLOWER]: Advancing to term 3
03:14:54.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.287261 5848 raft_consensus.cc:2468] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 79bba6a868c54f17a187e7418fe73c49 in term 3.
03:14:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.287873 5905 leader_election.cc:290] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [CANDIDATE]: Term 3 election: Requested vote from peers be6186a03e5f46e685128bf5ce84b5f4 (127.4.198.61:45443), cfc16134dcdc440a843537d928cf2c3e (127.4.198.60:37911)
03:14:54.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.289361 4919 leader_election.cc:304] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 79bba6a868c54f17a187e7418fe73c49, cfc16134dcdc440a843537d928cf2c3e; no voters: be6186a03e5f46e685128bf5ce84b5f4
03:14:54.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.290556 5905 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 3 FOLLOWER]: Leader election won for term 3
03:14:54.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.308581 5905 raft_consensus.cc:697] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [term 3 LEADER]: Becoming Leader. State: Replica: 79bba6a868c54f17a187e7418fe73c49, State: Running, Role: LEADER
03:14:54.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.309827 5905 consensus_queue.cc:237] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } }
03:14:54.312 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.313 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.313 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.313 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.334796 5910 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 79bba6a868c54f17a187e7418fe73c49. Latest consensus state: current_term: 3 leader_uuid: "79bba6a868c54f17a187e7418fe73c49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.335599 5910 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: This master's current role is: LEADER
03:14:54.339 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.339 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=6, TimeoutTracker(timeout=250, elapsed=206), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [3ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [6ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [23ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [25ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [41ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [44ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [46ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [47ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [61ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [114ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [114ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [121ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [126ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [134ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [134ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [181ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [185ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [207ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [207ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, deferred=Deferred@837675376(state=PENDING, result=null, callback=(continuation of Deferred@1270869928 after retry RPC after error@1616530125) -> (continuation of Deferred@1801152086 after retry RPC after error@899920932) -> (continuation of Deferred@30789527 after retry RPC after error@871096642) -> (continuation of Deferred@62019775 after retry RPC after error@723046009) -> (continuation of Deferred@69076474 after retry RPC after error@1754469834), errback=(continuation of Deferred@1270869928 after retry RPC after error@1616530125) -> (continuation of Deferred@1801152086 after retry RPC after error@899920932) -> (continuation of Deferred@30789527 after retry RPC after error@871096642) -> (continuation of Deferred@62019775 after retry RPC after error@723046009) -> (continuation of Deferred@69076474 after retry RPC after error@1754469834)))
03:14:54.340 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=6, TimeoutTracker(timeout=250, elapsed=206), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [3ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [6ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [23ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [25ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [41ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [44ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [46ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [47ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [61ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [114ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [114ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [121ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [126ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [134ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [134ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [181ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [185ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [207ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [207ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, deferred=Deferred@837675376(state=PENDING, result=null, callback=(continuation of Deferred@1270869928 after retry RPC after error@1616530125) -> (continuation of Deferred@1801152086 after retry RPC after error@899920932) -> (continuation of Deferred@30789527 after retry RPC after error@871096642) -> (continuation of Deferred@62019775 after retry RPC after error@723046009) -> (continuation of Deferred@69076474 after retry RPC after error@1754469834), errback=(continuation of Deferred@1270869928 after retry RPC after error@1616530125) -> (continuation of Deferred@1801152086 after retry RPC after error@899920932) -> (continuation of Deferred@30789527 after retry RPC after error@871096642) -> (continuation of Deferred@62019775 after retry RPC after error@723046009) -> (continuation of Deferred@69076474 after retry RPC after error@1754469834)))
03:14:54.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.348670 5912 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:14:54.352 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.353 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.353 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.353 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.362850 5912 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=01ca269f9a4146ff9be8addb9a2cc99e]
03:14:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.375520 5363 heartbeater.cc:344] Connected to a master server at 127.4.198.60:37911
03:14:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.379196 5912 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b]
03:14:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.380401 5838 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "823feaa369b047a89e2b9bdfaa6f3ca9" instance_seqno: 1773803684733554) as {username='slave'} at 127.4.198.1:51421; Asking this server to re-register.
03:14:54.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.381819 5363 heartbeater.cc:461] Registering TS with master...
03:14:54.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.385330 5838 ts_manager.cc:194] Registered new tserver with Master: 823feaa369b047a89e2b9bdfaa6f3ca9 (127.4.198.1:35799)
03:14:54.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.393739 5912 tablet_loader.cc:96] loaded metadata for tablet 66bcb4903dd74d08a0a8d147498e899b (table txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b])
03:14:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.396142 5912 tablet_loader.cc:96] loaded metadata for tablet 78974eeddc5f4534b2eeb0db74b16638 (table kudu_system.kudu_transactions [id=01ca269f9a4146ff9be8addb9a2cc99e])
03:14:54.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.398566 5912 tablet_loader.cc:96] loaded metadata for tablet 8d77b3eaef1d44f8842ab9fd4ce8b99d (table txn_manager_fallback_rolling [id=38d0a76607c74e47b30cf514a7631d2b])
03:14:54.401 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.402091 5912 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:14:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.411469 4917 consensus_peers.cc:597] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 -> Peer be6186a03e5f46e685128bf5ce84b5f4 (127.4.198.61:45443): Couldn't send request to peer be6186a03e5f46e685128bf5ce84b5f4. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:14:54.412 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.414 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.414 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.414 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.417372 5848 raft_consensus.cc:1275] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [term 3 FOLLOWER]: Refusing update from remote peer 79bba6a868c54f17a187e7418fe73c49: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
03:14:54.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.419104 5910 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
03:14:54.424 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.427134 5912 catalog_manager.cc:1269] Loaded cluster ID: 5aff527ef8e1466fb0288a917ecfa345
03:14:54.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.428524 5912 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:14:54.432 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.434 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.434 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.434 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.439399 5912 catalog_manager.cc:1514] Loading token signing keys...
03:14:54.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.448559 5912 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49: Loaded TSK: 0
03:14:54.459 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443; will retry
03:14:54.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.456073 5890 sys_catalog.cc:455] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 79bba6a868c54f17a187e7418fe73c49. Latest consensus state: current_term: 3 leader_uuid: "79bba6a868c54f17a187e7418fe73c49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:54.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.459967 5890 sys_catalog.cc:458] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: This master's current role is: FOLLOWER
03:14:54.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.464460 5912 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:14:54.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.465334 5890 sys_catalog.cc:455] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "79bba6a868c54f17a187e7418fe73c49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:54.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.466006 5890 sys_catalog.cc:458] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: This master's current role is: FOLLOWER
03:14:54.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.465885 5916 sys_catalog.cc:455] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "79bba6a868c54f17a187e7418fe73c49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:54.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.470989 5916 sys_catalog.cc:458] T 00000000000000000000000000000000 P cfc16134dcdc440a843537d928cf2c3e [sys.catalog]: This master's current role is: FOLLOWER
03:14:54.472 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.4.198.61:45443(127.4.198.61:45443)] connecting to peer
03:14:54.473 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.473 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.198.61:45443(127.4.198.61:45443)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.473 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.198.61:45443: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443
03:14:54.488 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:14:54.488 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=4, TimeoutTracker(timeout=250, elapsed=136), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [1ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [3ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [50ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [64ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [72ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [72ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [85ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [107ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [107ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [121ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [123ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [135ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, deferred=Deferred@2037764039(state=PENDING, result=null, callback=(continuation of Deferred@343129494 after retry RPC after error@1622559543) -> (continuation of Deferred@276512686 after retry RPC after error@1193791889) -> (continuation of Deferred@1320246254 after retry RPC after error@478961404), errback=(continuation of Deferred@343129494 after retry RPC after error@1622559543) -> (continuation of Deferred@276512686 after retry RPC after error@1193791889) -> (continuation of Deferred@1320246254 after retry RPC after error@478961404))) after lookup
03:14:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.502017 5910 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "79bba6a868c54f17a187e7418fe73c49" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79bba6a868c54f17a187e7418fe73c49" member_type: VOTER last_known_addr { host: "127.4.198.62" port: 43377 } } peers { permanent_uuid: "be6186a03e5f46e685128bf5ce84b5f4" member_type: VOTER last_known_addr { host: "127.4.198.61" port: 45443 } } peers { permanent_uuid: "cfc16134dcdc440a843537d928cf2c3e" member_type: VOTER last_known_addr { host: "127.4.198.60" port: 37911 } } }
03:14:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.502688 5910 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 [sys.catalog]: This master's current role is: LEADER
03:14:54.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.605904 4963 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:39910 (request call id 19) took 114 ms (client timeout 114 ms). Trace:
03:14:54.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.606331 4963 rpcz_store.cc:269] 0318 03:14:54.491252 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:14:54.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0318 03:14:54.492403 (+ 1151us) service_pool.cc:225] Handling call
03:14:54.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0318 03:14:54.605677 (+113274us) inbound_call.cc:173] Queueing success response
03:14:54.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":594,"init.queue_time_us":14249,"thread_start_us":52536,"threads_started":5}
03:14:54.609 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=256), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [1ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [3ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [50ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [64ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [72ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [72ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [85ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [107ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [107ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [121ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [123ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [135ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [137ms] sending RPC to server master-127.4.198.62:43377, [256ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@2037764039(state=PENDING, result=null, callback=(continuation of Deferred@343129494 after retry RPC after error@1622559543) -> (continuation of Deferred@276512686 after retry RPC after error@1193791889) -> (continuation of Deferred@1320246254 after retry RPC after error@478961404) -> (continuation of Deferred@1430492300 after retry RPC@739339500), errback=(continuation of Deferred@343129494 after retry RPC after error@1622559543) -> (continuation of Deferred@276512686 after retry RPC after error@1193791889) -> (continuation of Deferred@1320246254 after retry RPC after error@478961404) -> (continuation of Deferred@1430492300 after retry RPC@739339500)))
03:14:54.609 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=256), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [1ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [3ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [50ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [62ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [64ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [72ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [72ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [82ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [85ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [107ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [107ms] delaying RPC due to: Service unavailable: Master config (127.4.198.60:37911,127.4.198.62:43377,127.4.198.61:45443) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.60:37911, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.62:43377, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.198.61:45443, [121ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.61:45443: Network error: Failed to connect to peer master-127.4.198.61:45443(127.4.198.61:45443): Connection refused: /127.4.198.61:45443, [123ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.60:37911: OK, [135ms] Sub RPC ConnectToMaster: received response from server master-127.4.198.62:43377: OK, [137ms] sending RPC to server master-127.4.198.62:43377, [256ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@2037764039(state=PENDING, result=null, callback=(continuation of Deferred@343129494 after retry RPC after error@1622559543) -> (continuation of Deferred@276512686 after retry RPC after error@1193791889) -> (continuation of Deferred@1320246254 after retry RPC after error@478961404) -> (continuation of Deferred@1430492300 after retry RPC@739339500), errback=(continuation of Deferred@343129494 after retry RPC after error@1622559543) -> (continuation of Deferred@276512686 after retry RPC after error@1193791889) -> (continuation of Deferred@1320246254 after retry RPC after error@478961404) -> (continuation of Deferred@1430492300 after retry RPC@739339500)))
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.669901 4932 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39966:
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:14:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:14:54.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.730070 4932 catalog_manager.cc:7033] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:14:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.747263 5246 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.221s from last keepalive heartbeat (effective timeout is 1.000s)
03:14:54.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.785555 5257 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.4.198.60:37911)
03:14:54.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.811551 4963 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:39910 (request call id 20) took 199 ms (client timeout 250 ms). Trace:
03:14:54.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.811932 4963 rpcz_store.cc:269] 0318 03:14:54.612369 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:14:54.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0318 03:14:54.612495 (+ 126us) service_pool.cc:225] Handling call
03:14:54.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0318 03:14:54.811472 (+198977us) inbound_call.cc:173] Queueing success response
03:14:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
03:14:54.813 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 823feaa369b047a89e2b9bdfaa6f3ca9 (127.4.198.1:35799): transaction ID 0 is not available for further commits: state: ABORT_IN_PROGRESS user: "slave" start_timestamp: 1773803689 last_transition_timestamp: 1773803694
03:14:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.873871 5928 tablet_replica.cc:1307] Aborted: operation has been aborted: cancelling pending write operations
03:14:54.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.950501 5838 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5b6e1325a34b4f2c88e8626a3c208d6e" instance_seqno: 1773803686544691) as {username='slave'} at 127.4.198.2:52021; Asking this server to re-register.
03:14:54.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:54.952704 4917 consensus_peers.cc:597] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 -> Peer be6186a03e5f46e685128bf5ce84b5f4 (127.4.198.61:45443): Couldn't send request to peer be6186a03e5f46e685128bf5ce84b5f4. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:14:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.958407 5511 heartbeater.cc:461] Registering TS with master...
03:14:54.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:54.963135 5838 ts_manager.cc:194] Registered new tserver with Master: 5b6e1325a34b4f2c88e8626a3c208d6e (127.4.198.2:39509)
03:14:55.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:55.047509 4917 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111) [suppressed 8 similar messages]
03:14:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:55.372395 5365 heartbeater.cc:499] Master 127.4.198.62:43377 was elected leader, sending a full tablet report...
03:14:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:55.467937 4917 consensus_peers.cc:597] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 -> Peer be6186a03e5f46e685128bf5ce84b5f4 (127.4.198.61:45443): Couldn't send request to peer be6186a03e5f46e685128bf5ce84b5f4. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
03:14:55.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:55.871150 5513 heartbeater.cc:499] Master 127.4.198.62:43377 was elected leader, sending a full tablet report...
03:14:55.882 [DEBUG - main] (Connection.java:575) [peer master-127.4.198.61:45443(127.4.198.61:45443)] disconnecting while in state TERMINATED
03:14:55.882 [DEBUG - main] (Connection.java:575) [peer master-127.4.198.60:37911(127.4.198.60:37911)] disconnecting while in state READY
03:14:55.883 [DEBUG - main] (Connection.java:575) [peer master-127.4.198.62:43377(127.4.198.62:43377)] disconnecting while in state READY
03:14:55.883 [DEBUG - main] (Connection.java:575) [peer 27a6991a885b47e594fb08f14be5a0f0(127.4.198.3:42651)] disconnecting while in state READY
03:14:55.883 [DEBUG - kudu-nio-7] (Connection.java:239) [peer master-127.4.198.60:37911(127.4.198.60:37911)] handling channelInactive
03:14:55.883 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.4.198.60:37911(127.4.198.60:37911)] cleaning up while in state READY due to: connection closed
03:14:55.884 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 27a6991a885b47e594fb08f14be5a0f0(127.4.198.3:42651)] handling channelInactive
03:14:55.884 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 27a6991a885b47e594fb08f14be5a0f0(127.4.198.3:42651)] cleaning up while in state READY due to: connection closed
03:14:55.885 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:14:55.886 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.4.198.62:43377(127.4.198.62:43377)] handling channelInactive
03:14:55.886 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.198.62:43377(127.4.198.62:43377)] cleaning up while in state READY due to: connection closed
03:14:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:55.887698 4888 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu with pid 5220
03:14:55.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:55.944739 5537 connection.cc:537] client connection to 127.4.198.1:35799 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:14:55.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:55.945217 5537 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:14:55.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:55.964319 4888 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu with pid 5368
03:14:55.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:55.989363 4917 consensus_peers.cc:597] T 00000000000000000000000000000000 P 79bba6a868c54f17a187e7418fe73c49 -> Peer be6186a03e5f46e685128bf5ce84b5f4 (127.4.198.61:45443): Couldn't send request to peer be6186a03e5f46e685128bf5ce84b5f4. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.61:45443: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
03:14:56.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:55.999993 5537 connection.cc:537] server connection from 127.4.198.1:39187 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:14:56.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260318 03:14:56.001562 5537 consensus_peers.cc:597] T 66bcb4903dd74d08a0a8d147498e899b P 27a6991a885b47e594fb08f14be5a0f0 -> Peer 823feaa369b047a89e2b9bdfaa6f3ca9 (127.4.198.1:35799): Couldn't send request to peer 823feaa369b047a89e2b9bdfaa6f3ca9. Status: Network error: Client connection negotiation failed: client connection to 127.4.198.1:35799: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:14:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:56.005015 4888 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu with pid 5516
03:14:56.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:56.033691 4888 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu with pid 4900
03:14:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260318 03:14:56.074944 4888 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskaVXZmI/build/tsan/bin/kudu with pid 5798
03:14:57.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-18T03:14:57Z chronyd exiting
03:14:58.173 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)