09:15:05.338 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
09:15:05.338 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskapZmwC/test-tmp
09:15:05.338 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
09:15:05.338 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskapZmwC/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
09:15:05.340 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
09:15:05.671 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:15:05.671 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:15:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:05.674732 16625 mini_oidc.cc:150] Starting JWKS server
09:15:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:05.678061 16625 webserver.cc:492] Webserver started at http://127.0.0.1:43617/ using document root <none> and password file <none>
09:15:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:05.678465 16625 mini_oidc.cc:186] Starting OIDC Discovery server
09:15:05.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:05.680122 16625 webserver.cc:492] Webserver started at http://127.0.0.1:43709/ using document root <none> and password file <none>
09:15:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-30T09:15:05Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:15:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-30T09:15:05Z Disabled control of system clock
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:05.740242 16625 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/wal
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/data
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/logs
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/data/info.pb
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.126:39927
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.126
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:15:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:15:06.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.038329 16637 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:15:06.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.038996 16637 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:15:06.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.039505 16637 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:15:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.071206 16637 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:15:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.071522 16637 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:15:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.071691 16637 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:15:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.071892 16637 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:15:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.072090 16637 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:15:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.072288 16637 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:15:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.072438 16637 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43617/jwks.json
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:06.108342 16637 master_runner.cc:386] Master server non-default flags:
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/data
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/wal
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.126:39927
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/data/info.pb
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.126
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/logs
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Jan 2026 08:43:23 UTC on 5fd53c4cbb9d
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10261
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:15:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:06.109685 16637 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:15:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:06.111677 16637 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:15:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.122915 16643 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:07.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:06.124339 16644 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:07.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.436589 16646 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:07.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.439067 16637 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.316s user 0.415s sys 0.895s
09:15:07.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.439630 16637 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.316s user 0.416s sys 0.896s
09:15:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.441598 16645 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1310 milliseconds
09:15:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.441659 16637 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:15:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.442991 16637 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:15:07.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.452577 16637 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:15:07.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.454010 16637 hybrid_clock.cc:648] HybridClock initialized: now 1769764507453945 us; error 87 us; skew 500 ppm
09:15:07.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.454819 16637 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:15:07.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.462028 16637 webserver.cc:492] Webserver started at http://127.16.60.126:40961/ using document root <none> and password file <none>
09:15:07.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.462908 16637 fs_manager.cc:362] Metadata directory not provided
09:15:07.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.463094 16637 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:15:07.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.463500 16637 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
09:15:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.468014 16637 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/data/instance:
09:15:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661dc84b98204b03aaa17fef82fe8abc"
09:15:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:07 on dist-test-slave-0ffw"
09:15:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.469036 16637 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/wal/instance:
09:15:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661dc84b98204b03aaa17fef82fe8abc"
09:15:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:07 on dist-test-slave-0ffw"
09:15:07.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.476382 16637 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.005s
09:15:07.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.482533 16653 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:15:07.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.483743 16637 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
09:15:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.484095 16637 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/data,/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/wal
09:15:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661dc84b98204b03aaa17fef82fe8abc"
09:15:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:07 on dist-test-slave-0ffw"
09:15:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.484433 16637 fs_report.cc:389] FS layout report
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/wal
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/wal
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/data/data
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:15:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:15:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.568042 16637 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:15:07.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.569573 16637 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:15:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.570055 16637 kserver.cc:163] Server-wide thread pool size limit: 3276
09:15:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.651633 16637 rpc_server.cc:307] RPC server started. Bound to: 127.16.60.126:39927
09:15:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.651749 16714 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.60.126:39927 every 8 connection(s)
09:15:07.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.654492 16637 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/data/info.pb
09:15:07.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.659927 16625 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu as pid 16637
09:15:07.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.660310 16625 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-0/wal/instance
09:15:07.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.661000 16715 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.663627 16625 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/logs
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data/info.pb
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.125:32949
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.125
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:15:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:15:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
09:15:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:15:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:15:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.673522 16715 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } has no permanent_uuid. Determining permanent_uuid...
09:15:07.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.702068 16715 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } has no permanent_uuid. Determining permanent_uuid...
09:15:07.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.704187 16654 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111)
09:15:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.708284 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.125:32949: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111)
09:15:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.757844 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } attempt: 1
09:15:07.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.761912 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.125:32949: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111)
09:15:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.830425 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } attempt: 2
09:15:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.834486 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.125:32949: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111)
09:15:07.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:07.925956 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } attempt: 3
09:15:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.930358 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.125:32949: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111)
09:15:07.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.968575 16717 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:15:07.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.969101 16717 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:15:07.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:07.969708 16717 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:15:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.001474 16717 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:15:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.001767 16717 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:15:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.001941 16717 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:15:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.002139 16717 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:15:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.002341 16717 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:15:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.002552 16717 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:15:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.002712 16717 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43617/jwks.json
09:15:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:08.038343 16717 master_runner.cc:386] Master server non-default flags:
09:15:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:15:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data
09:15:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal
09:15:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:15:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.125:32949
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data/info.pb
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.125
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/logs
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Jan 2026 08:43:23 UTC on 5fd53c4cbb9d
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10261
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:15:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:08.039638 16717 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:15:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:08.041378 16717 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:15:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.051963 16726 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:08.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:08.073851 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } attempt: 4
09:15:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.081822 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.125:32949: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111)
09:15:08.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:08.381449 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } attempt: 5
09:15:08.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.389227 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.125:32949: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111)
09:15:08.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:08.937453 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } attempt: 6
09:15:08.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.961856 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.125:32949: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111)
09:15:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:08.053614 16727 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:09.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.382334 16728 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1324 milliseconds
09:15:09.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.382418 16717 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.330s user 0.448s sys 0.873s
09:15:09.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.382942 16717 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.330s user 0.449s sys 0.873s
09:15:09.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.384100 16729 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:09.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.384166 16717 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:15:09.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.385437 16717 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:15:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.388037 16717 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:15:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.389464 16717 hybrid_clock.cc:648] HybridClock initialized: now 1769764509389389 us; error 65 us; skew 500 ppm
09:15:09.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.390313 16717 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:15:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.397616 16717 webserver.cc:492] Webserver started at http://127.16.60.125:42103/ using document root <none> and password file <none>
09:15:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.398574 16717 fs_manager.cc:362] Metadata directory not provided
09:15:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.398818 16717 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:15:09.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.399303 16717 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
09:15:09.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.403635 16717 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data/instance:
09:15:09.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4"
09:15:09.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:09 on dist-test-slave-0ffw"
09:15:09.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.404757 16717 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal/instance:
09:15:09.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4"
09:15:09.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:09 on dist-test-slave-0ffw"
09:15:09.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.412498 16717 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
09:15:09.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.418645 16738 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:15:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.419808 16717 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
09:15:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.420087 16717 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data,/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal
09:15:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4"
09:15:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:09 on dist-test-slave-0ffw"
09:15:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.420372 16717 fs_report.cc:389] FS layout report
09:15:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:15:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal
09:15:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal
09:15:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data/data
09:15:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:15:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:15:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:15:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:15:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:15:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:15:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:15:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:15:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:15:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:15:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:15:09.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.507961 16717 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:15:09.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.509418 16717 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:15:09.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.509892 16717 kserver.cc:163] Server-wide thread pool size limit: 3276
09:15:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.593005 16717 rpc_server.cc:307] RPC server started. Bound to: 127.16.60.125:32949
09:15:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.593119 16799 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.60.125:32949 every 8 connection(s)
09:15:09.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.595660 16717 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data/info.pb
09:15:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.601557 16800 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:09.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.605965 16625 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu as pid 16717
09:15:09.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.606458 16625 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal/instance
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.609616 16625 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/logs
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data/info.pb
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.124:44777
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.124
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:15:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:15:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.616561 16800 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } has no permanent_uuid. Determining permanent_uuid...
09:15:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.637336 16800 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } has no permanent_uuid. Determining permanent_uuid...
09:15:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.652029 16800 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } has no permanent_uuid. Determining permanent_uuid...
09:15:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.654850 16742 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:09.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.659345 16800 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:09.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.708879 16800 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 1
09:15:09.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.712816 16800 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:09.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.781313 16800 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 2
09:15:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.785418 16800 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:09.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.876837 16800 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 3
09:15:09.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.880612 16800 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.918778 16802 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:15:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.919319 16802 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:15:09.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.919816 16802 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:15:09.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.952452 16802 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:15:09.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.952740 16802 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:15:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.952906 16802 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:15:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.953099 16802 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:15:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.953294 16802 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:15:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.953514 16802 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:15:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:09.953663 16802 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43617/jwks.json
09:15:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.989204 16802 master_runner.cc:386] Master server non-default flags:
09:15:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:15:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data
09:15:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal
09:15:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.124:44777
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data/info.pb
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.124
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/logs
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Jan 2026 08:43:23 UTC on 5fd53c4cbb9d
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10261
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:15:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.990506 16802 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:15:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:09.992208 16802 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:15:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:10.002467 16812 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:10.022423 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } attempt: 7
09:15:10.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:10.024056 16800 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 4
09:15:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:10.030851 16800 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:10.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:10.052243 16715 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } has no permanent_uuid. Determining permanent_uuid...
09:15:10.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:10.079279 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:10.137840 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 1
09:15:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:10.145545 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:10.227139 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 2
09:15:10.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:10.234687 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:10.320252 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 3
09:15:10.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:10.327204 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:10.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:10.330390 16800 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 5
09:15:10.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:10.337380 16800 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:10.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:10.467787 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 4
09:15:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:10.476166 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:10.759923 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 5
09:15:10.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:10.768792 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:10.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:10.884912 16800 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 6
09:15:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:10.910750 16800 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:11.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.321354 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 6
09:15:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:11.344004 16715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.60.124:44777: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:11.407227 16811 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 16802
09:15:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:11.534247 16811 kernel_stack_watchdog.cc:198] Thread 16802 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
09:15:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:15:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:15:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:15:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:15:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:10.004165 16813 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:11.535436 16802 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.532s user 0.002s sys 0.000s
09:15:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:11.535845 16802 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.533s user 0.002s sys 0.001s
09:15:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:11.535874 16814 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1531 milliseconds
09:15:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:11.537664 16815 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:11.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.537590 16802 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:15:11.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.540445 16802 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:15:11.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.542869 16802 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:15:11.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.544205 16802 hybrid_clock.cc:648] HybridClock initialized: now 1769764511544162 us; error 49 us; skew 500 ppm
09:15:11.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.544979 16802 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:15:11.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.551614 16802 webserver.cc:492] Webserver started at http://127.16.60.124:42633/ using document root <none> and password file <none>
09:15:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.552557 16802 fs_manager.cc:362] Metadata directory not provided
09:15:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.552748 16802 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:15:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.553143 16802 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
09:15:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.557832 16802 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data/instance:
09:15:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c796ccb9931245c8a473b1b23e405086"
09:15:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:11 on dist-test-slave-0ffw"
09:15:11.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.558857 16802 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal/instance:
09:15:11.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c796ccb9931245c8a473b1b23e405086"
09:15:11.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:11 on dist-test-slave-0ffw"
09:15:11.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.565898 16802 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
09:15:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.572242 16829 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:15:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.573601 16802 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
09:15:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.573904 16802 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data,/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal
09:15:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c796ccb9931245c8a473b1b23e405086"
09:15:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:11 on dist-test-slave-0ffw"
09:15:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.574187 16802 fs_report.cc:389] FS layout report
09:15:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:15:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal
09:15:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal
09:15:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data/data
09:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:15:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.629029 16802 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:15:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.630568 16802 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:15:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.630950 16802 kserver.cc:163] Server-wide thread pool size limit: 3276
09:15:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.714571 16802 rpc_server.cc:307] RPC server started. Bound to: 127.16.60.124:44777
09:15:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.714643 16890 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.60.124:44777 every 8 connection(s)
09:15:11.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.717116 16802 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data/info.pb
09:15:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.722832 16891 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.723641 16625 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu as pid 16802
09:15:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.723999 16625 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal/instance
09:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.735952 16891 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } has no permanent_uuid. Determining permanent_uuid...
09:15:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.764928 16891 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } has no permanent_uuid. Determining permanent_uuid...
09:15:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.775305 16891 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } has no permanent_uuid. Determining permanent_uuid...
09:15:11.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.790983 16891 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: Bootstrap starting.
09:15:11.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.797255 16891 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: Neither blocks nor log segments found. Creating new log.
09:15:11.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.799012 16891 log.cc:826] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: Log is configured to *not* fsync() on all Append() calls
09:15:11.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.803757 16891 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: No bootstrap required, opened a new log
09:15:11.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.822386 16891 raft_consensus.cc:359] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:11.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.823313 16891 raft_consensus.cc:385] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:11.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.823607 16891 raft_consensus.cc:740] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c796ccb9931245c8a473b1b23e405086, State: Initialized, Role: FOLLOWER
09:15:11.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.824353 16891 consensus_queue.cc:260] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.826581 16900 sys_catalog.cc:455] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.827138 16900 sys_catalog.cc:458] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: This master's current role is: FOLLOWER
09:15:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.828157 16891 sys_catalog.cc:565] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: configured and running, proceeding with master startup.
09:15:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.829636 16900 raft_consensus.cc:493] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:15:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.829984 16900 raft_consensus.cc:515] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.834290 16900 leader_election.cc:290] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927), d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949)
09:15:11.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:11.859496 16911 catalog_manager.cc:1568] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:15:11.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:11.859838 16911 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:15:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.851987 16802 master_runner.cc:186] Error getting master registration for 127.16.60.126:39927: OK, instance_id { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" instance_seqno: 1769764507633304 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:15:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.881662 16802 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:15:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:11.885828 16835 tablet.cc:2378] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:15:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.971257 16800 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 7
09:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.990581 16800 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: Bootstrap starting.
09:15:11.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:11.997866 16800 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: Neither blocks nor log segments found. Creating new log.
09:15:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.000093 16800 log.cc:826] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: Log is configured to *not* fsync() on all Append() calls
09:15:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.006152 16800 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: No bootstrap required, opened a new log
09:15:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.034914 16800 raft_consensus.cc:359] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.036015 16800 raft_consensus.cc:385] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.036485 16800 raft_consensus.cc:740] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9e5dacfdeb94d4d8bf1f17195e777b4, State: Initialized, Role: FOLLOWER
09:15:12.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.037479 16800 consensus_queue.cc:260] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:12.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.040467 16915 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:12.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.041435 16915 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: This master's current role is: FOLLOWER
09:15:12.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.042668 16800 sys_catalog.cc:565] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: configured and running, proceeding with master startup.
09:15:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.042811 16765 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c796ccb9931245c8a473b1b23e405086" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" is_pre_election: true
09:15:12.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.043601 16765 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c796ccb9931245c8a473b1b23e405086 in term 0.
09:15:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.044971 16830 leader_election.cc:304] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c796ccb9931245c8a473b1b23e405086, d9e5dacfdeb94d4d8bf1f17195e777b4; no voters:
09:15:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.045850 16900 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:15:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.046149 16900 raft_consensus.cc:493] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:15:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.046447 16900 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 0 FOLLOWER]: Advancing to term 1
09:15:12.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.051273 16900 raft_consensus.cc:515] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:12.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.053757 16900 leader_election.cc:290] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [CANDIDATE]: Term 1 election: Requested vote from peers 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927), d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949)
09:15:12.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.054488 16765 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c796ccb9931245c8a473b1b23e405086" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4"
09:15:12.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.054991 16765 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 0 FOLLOWER]: Advancing to term 1
09:15:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.068121 16765 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c796ccb9931245c8a473b1b23e405086 in term 1.
09:15:12.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.069120 16830 leader_election.cc:304] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c796ccb9931245c8a473b1b23e405086, d9e5dacfdeb94d4d8bf1f17195e777b4; no voters:
09:15:12.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.070209 16900 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 1 FOLLOWER]: Leader election won for term 1
09:15:12.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.072072 16900 raft_consensus.cc:697] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 1 LEADER]: Becoming Leader. State: Replica: c796ccb9931245c8a473b1b23e405086, State: Running, Role: LEADER
09:15:12.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.072916 16900 consensus_queue.cc:237] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:12.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.092747 16921 sys_catalog.cc:455] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c796ccb9931245c8a473b1b23e405086. Latest consensus state: current_term: 1 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:12.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.095069 16921 sys_catalog.cc:458] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: This master's current role is: LEADER
09:15:12.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.099556 16928 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:15:12.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.107014 16928 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:15:12.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.134785 16765 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 1 FOLLOWER]: Refusing update from remote peer c796ccb9931245c8a473b1b23e405086: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:15:12.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.136811 16900 consensus_queue.cc:1048] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:15:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.112510 16717 master_runner.cc:186] Error getting master registration for 127.16.60.126:39927: OK, instance_id { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" instance_seqno: 1769764507633304 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:15:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.150497 16717 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:15:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.153406 16929 catalog_manager.cc:1568] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:15:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.154127 16929 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:15:12.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.184408 16915 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c796ccb9931245c8a473b1b23e405086. Latest consensus state: current_term: 1 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:12.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.185655 16915 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: This master's current role is: FOLLOWER
09:15:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.189376 16921 sys_catalog.cc:455] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.190187 16921 sys_catalog.cc:458] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: This master's current role is: LEADER
09:15:12.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.193619 16930 mvcc.cc:204] Tried to move back new op lower bound from 7248955441685110784 to 7248955441460256768. Current Snapshot: MvccSnapshot[applied={T|T < 7248955441685110784}]
09:15:12.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.202277 16928 catalog_manager.cc:1357] Generated new cluster ID: 91258d94f6904e95815ac3730f414487
09:15:12.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.202557 16928 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:15:12.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.203034 16915 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:12.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.203675 16915 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: This master's current role is: FOLLOWER
09:15:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.217034 16832 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.16.60.126:39927 timed out after 0.384s (SENT)
09:15:12.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.244881 16928 catalog_manager.cc:1380] Generated new certificate authority record
09:15:12.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.251044 16928 catalog_manager.cc:1514] Loading token signing keys...
09:15:12.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.291721 16928 catalog_manager.cc:6027] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: Generated new TSK 0
09:15:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.296325 16928 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:15:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.344282 16625 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/wal
09:15:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/data
09:15:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/logs
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/data/info.pb
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:15:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.65:0
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.60.65
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.65
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:15:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:15:12.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.377416 16715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } attempt: 7
09:15:12.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.428120 16715 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc: Bootstrap starting.
09:15:12.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.444717 16715 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc: Neither blocks nor log segments found. Creating new log.
09:15:12.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.447068 16715 log.cc:826] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc: Log is configured to *not* fsync() on all Append() calls
09:15:12.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.457775 16715 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc: No bootstrap required, opened a new log
09:15:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.562203 16715 raft_consensus.cc:359] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.563143 16715 raft_consensus.cc:385] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.563474 16715 raft_consensus.cc:740] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 661dc84b98204b03aaa17fef82fe8abc, State: Initialized, Role: FOLLOWER
09:15:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.564397 16715 consensus_queue.cc:260] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:12.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.584302 16715 sys_catalog.cc:565] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: configured and running, proceeding with master startup.
09:15:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.585088 16677 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c796ccb9931245c8a473b1b23e405086" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "661dc84b98204b03aaa17fef82fe8abc" is_pre_election: true
09:15:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.585938 16677 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c796ccb9931245c8a473b1b23e405086 in term 0.
09:15:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.586481 16677 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:47790 (request call id 5) took 353 ms (client timeout 384 ms). Trace:
09:15:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.586855 16677 rpcz_store.cc:269] 0130 09:15:12.232819 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:15:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:12.232934 (+ 115us) service_pool.cc:225] Handling call
09:15:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:12.586413 (+353479us) inbound_call.cc:173] Queueing success response
09:15:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:15:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.589265 16679 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c796ccb9931245c8a473b1b23e405086" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "661dc84b98204b03aaa17fef82fe8abc"
09:15:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.590045 16679 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 0 FOLLOWER]: Advancing to term 1
09:15:12.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.593972 16676 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c796ccb9931245c8a473b1b23e405086" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "661dc84b98204b03aaa17fef82fe8abc"
09:15:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.584452 16680 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c796ccb9931245c8a473b1b23e405086" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "661dc84b98204b03aaa17fef82fe8abc" is_pre_election: true
09:15:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.598533 16680 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:47790 (request call id 1) took 765 ms (client timeout 384 ms). Trace:
09:15:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.598866 16680 rpcz_store.cc:269] 0130 09:15:11.832908 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:15:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:11.833251 (+ 343us) service_pool.cc:225] Handling call
09:15:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:12.598464 (+765213us) inbound_call.cc:173] Queueing success response
09:15:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:15:12.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.604131 16679 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c796ccb9931245c8a473b1b23e405086 in term 1.
09:15:12.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.605396 16678 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 1 FOLLOWER]: Refusing update from remote peer c796ccb9931245c8a473b1b23e405086: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:15:12.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.610421 16921 consensus_queue.cc:1048] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [LEADER]: Connected to new peer: Peer: permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
09:15:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.617480 16679 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:47790 (request call id 3) took 564 ms (client timeout 416 ms). Trace:
09:15:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.618453 16679 rpcz_store.cc:269] 0130 09:15:12.053420 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:15:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:12.053583 (+ 163us) service_pool.cc:225] Handling call
09:15:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:12.617415 (+563832us) inbound_call.cc:173] Queueing success response
09:15:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:15:12.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.694989 16941 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:12.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.695331 16948 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:12.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.695885 16941 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: This master's current role is: FOLLOWER
09:15:12.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.696146 16948 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: This master's current role is: FOLLOWER
09:15:12.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.697419 16943 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: SysCatalogTable state changed. Reason: New leader c796ccb9931245c8a473b1b23e405086. Latest consensus state: current_term: 1 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:12.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.698325 16943 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: This master's current role is: FOLLOWER
09:15:12.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.708608 16921 sys_catalog.cc:455] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:12.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.709527 16921 sys_catalog.cc:458] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: This master's current role is: LEADER
09:15:12.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:12.750624 16958 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:15:12.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.879091 16938 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:15:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.879946 16938 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:15:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.882814 16938 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:15:12.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.944370 16938 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:15:12.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.945500 16938 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:15:12.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.945809 16938 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:15:12.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.946076 16938 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:15:12.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:12.946851 16938 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.60.65
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:13.012754 16938 tablet_server_runner.cc:78] Tablet server non-default flags:
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/data
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/wal
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.65:0
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/data/info.pb
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.65
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:15:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.60.65
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/logs
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Jan 2026 08:43:23 UTC on 5fd53c4cbb9d
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10261
09:15:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:15:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:13.014487 16938 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:15:13.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:13.016665 16938 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:15:13.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:13.033566 16964 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:13.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:13.160177 16929 catalog_manager.cc:1269] Loaded cluster ID: 91258d94f6904e95815ac3730f414487
09:15:13.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:13.160676 16929 catalog_manager.cc:1562] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: loading cluster ID for follower catalog manager: success
09:15:13.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:13.173059 16929 catalog_manager.cc:1584] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: acquiring CA information for follower catalog manager: success
09:15:13.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:13.187804 16929 catalog_manager.cc:1612] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:15:13.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:13.781548 16958 catalog_manager.cc:1269] Loaded cluster ID: 91258d94f6904e95815ac3730f414487
09:15:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:13.781853 16958 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc: loading cluster ID for follower catalog manager: success
09:15:13.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:13.805979 16958 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc: acquiring CA information for follower catalog manager: success
09:15:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:13.811250 16958 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:15:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:14.434054 16963 debug-util.cc:398] Leaking SignalData structure 0x7b0800037660 after lost signal to thread 16938
09:15:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:15.346958 16963 kernel_stack_watchdog.cc:198] Thread 16938 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
09:15:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:15:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:15:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:15:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:15:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:13.034193 16965 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:15.352614 16938 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.321s user 0.777s sys 1.396s
09:15:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:15.352962 16938 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.322s user 0.777s sys 1.396s
09:15:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:15.364924 16972 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.365971 16938 server_base.cc:1060] running on GCE node
09:15:15.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.367270 16938 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:15:15.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.375700 16938 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:15:15.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.379290 16938 hybrid_clock.cc:648] HybridClock initialized: now 1769764515379242 us; error 40 us; skew 500 ppm
09:15:15.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.380344 16938 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:15:15.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.402421 16938 webserver.cc:492] Webserver started at http://127.16.60.65:37537/ using document root <none> and password file <none>
09:15:15.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.403658 16938 fs_manager.cc:362] Metadata directory not provided
09:15:15.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.403944 16938 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:15:15.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.404472 16938 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
09:15:15.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.410874 16938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/data/instance:
09:15:15.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1aefced973a34ed69a4c40fe377b10aa"
09:15:15.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:15 on dist-test-slave-0ffw"
09:15:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.412484 16938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/wal/instance:
09:15:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1aefced973a34ed69a4c40fe377b10aa"
09:15:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:15 on dist-test-slave-0ffw"
09:15:15.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.427412 16938 fs_manager.cc:696] Time spent creating directory manager: real 0.014s user 0.011s sys 0.002s
09:15:15.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.440922 16977 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:15:15.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.442268 16938 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.008s sys 0.001s
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.442642 16938 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/data,/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/wal
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1aefced973a34ed69a4c40fe377b10aa"
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:15 on dist-test-slave-0ffw"
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.443095 16938 fs_report.cc:389] FS layout report
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/wal
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/wal
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/data/data
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:15:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:15:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:15:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:15:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:15:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:15:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:15:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:15:15.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.555465 16938 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:15:15.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.557542 16938 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:15:15.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.558079 16938 kserver.cc:163] Server-wide thread pool size limit: 3276
09:15:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.580374 16938 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:15:15.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.580675 16938 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:15:15.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.580978 16938 ts_tablet_manager.cc:616] Registered 0 tablets
09:15:15.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.581183 16938 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:15:15.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.955155 16938 rpc_server.cc:307] RPC server started. Bound to: 127.16.60.65:35089
09:15:15.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.955265 17103 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.60.65:35089 every 8 connection(s)
09:15:15.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.958992 16938 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/data/info.pb
09:15:15.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.964785 16625 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu as pid 16938
09:15:15.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.965409 16625 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-0/wal/instance
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.983721 16625 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/wal
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/data
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/logs
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/data/info.pb
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.66:0
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.60.66
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.66
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:15:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:15:15.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.990479 17104 heartbeater.cc:344] Connected to a master server at 127.16.60.124:44777
09:15:15.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.992013 17104 heartbeater.cc:461] Registering TS with master...
09:15:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:15.999647 17104 heartbeater.cc:507] Master 127.16.60.124:44777 requested a full tablet report, sending...
09:15:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.000379 17107 heartbeater.cc:344] Connected to a master server at 127.16.60.126:39927
09:15:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.000763 17107 heartbeater.cc:461] Registering TS with master...
09:15:16.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.001775 17107 heartbeater.cc:507] Master 127.16.60.126:39927 requested a full tablet report, sending...
09:15:16.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.002970 16846 ts_manager.cc:194] Registered new tserver with Master: 1aefced973a34ed69a4c40fe377b10aa (127.16.60.65:35089)
09:15:16.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.004621 16670 ts_manager.cc:194] Registered new tserver with Master: 1aefced973a34ed69a4c40fe377b10aa (127.16.60.65:35089)
09:15:16.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.006021 16846 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.60.65:47269
09:15:16.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.013792 17105 heartbeater.cc:344] Connected to a master server at 127.16.60.125:32949
09:15:16.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.014835 17105 heartbeater.cc:461] Registering TS with master...
09:15:16.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.015496 17105 heartbeater.cc:507] Master 127.16.60.125:32949 requested a full tablet report, sending...
09:15:16.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.018446 16755 ts_manager.cc:194] Registered new tserver with Master: 1aefced973a34ed69a4c40fe377b10aa (127.16.60.65:35089)
09:15:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:16.380792 17109 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:15:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:16.381274 17109 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:15:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:16.381916 17109 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:15:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:16.416006 17109 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:15:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:16.416677 17109 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:15:16.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:16.416877 17109 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:15:16.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:16.417033 17109 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:15:16.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:16.417447 17109 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.60.66
09:15:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.454533 17109 tablet_server_runner.cc:78] Tablet server non-default flags:
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/data
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/wal
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.66:0
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/data/info.pb
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.66
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.60.66
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/logs
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:15:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
09:15:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:15:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Jan 2026 08:43:23 UTC on 5fd53c4cbb9d
09:15:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10261
09:15:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:15:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.456171 17109 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:15:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:16.457824 17109 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:15:16.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:16.471489 17116 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:17.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.015576 17104 heartbeater.cc:499] Master 127.16.60.124:44777 was elected leader, sending a full tablet report...
09:15:17.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:16.473881 17117 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:17.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:16.475688 17119 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:17.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:17.867655 17118 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1392 milliseconds
09:15:17.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.867766 17109 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:15:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.868996 17109 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:15:17.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.871974 17109 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:15:17.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.873453 17109 hybrid_clock.cc:648] HybridClock initialized: now 1769764517873428 us; error 45 us; skew 500 ppm
09:15:17.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.874248 17109 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:15:17.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.880558 17109 webserver.cc:492] Webserver started at http://127.16.60.66:42831/ using document root <none> and password file <none>
09:15:17.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.881528 17109 fs_manager.cc:362] Metadata directory not provided
09:15:17.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.881728 17109 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:15:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.882121 17109 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
09:15:17.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.886463 17109 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/data/instance:
09:15:17.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524"
09:15:17.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:17 on dist-test-slave-0ffw"
09:15:17.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.887539 17109 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/wal/instance:
09:15:17.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524"
09:15:17.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:17 on dist-test-slave-0ffw"
09:15:17.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.895058 17109 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
09:15:17.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.900725 17127 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:15:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.901822 17109 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.001s
09:15:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.902107 17109 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/data,/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/wal
09:15:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524"
09:15:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:17 on dist-test-slave-0ffw"
09:15:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.902434 17109 fs_report.cc:389] FS layout report
09:15:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:15:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/wal
09:15:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/wal
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/data/data
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:15:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:15:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.964360 17109 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:15:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.966668 17109 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:15:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.967216 17109 kserver.cc:163] Server-wide thread pool size limit: 3276
09:15:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.978466 17109 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:15:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.978731 17109 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:15:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.979071 17109 ts_tablet_manager.cc:616] Registered 0 tablets
09:15:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:17.979266 17109 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:15:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.172580 17109 rpc_server.cc:307] RPC server started. Bound to: 127.16.60.66:34795
09:15:18.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.173061 17252 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.60.66:34795 every 8 connection(s)
09:15:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.175204 17109 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/data/info.pb
09:15:18.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.180207 16625 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu as pid 17109
09:15:18.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.180680 16625 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-1/wal/instance
09:15:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.190713 17255 heartbeater.cc:344] Connected to a master server at 127.16.60.126:39927
09:15:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.191395 17255 heartbeater.cc:461] Registering TS with master...
09:15:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.192909 17255 heartbeater.cc:507] Master 127.16.60.126:39927 requested a full tablet report, sending...
09:15:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.193123 16625 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/wal
09:15:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/data
09:15:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:15:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:15:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:15:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/logs
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/data/info.pb
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.67:0
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.60.67
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.67
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:15:18.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:15:18.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.196310 16670 ts_manager.cc:194] Registered new tserver with Master: ac5023c640ae4ccdb6e27ef5cfb9c524 (127.16.60.66:34795)
09:15:18.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.199362 17253 heartbeater.cc:344] Connected to a master server at 127.16.60.124:44777
09:15:18.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.200403 17253 heartbeater.cc:461] Registering TS with master...
09:15:18.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.201380 17253 heartbeater.cc:507] Master 127.16.60.124:44777 requested a full tablet report, sending...
09:15:18.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.203704 16846 ts_manager.cc:194] Registered new tserver with Master: ac5023c640ae4ccdb6e27ef5cfb9c524 (127.16.60.66:34795)
09:15:18.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.205317 16846 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.60.66:59681
09:15:18.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.206637 17254 heartbeater.cc:344] Connected to a master server at 127.16.60.125:32949
09:15:18.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.207007 17254 heartbeater.cc:461] Registering TS with master...
09:15:18.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.207901 17254 heartbeater.cc:507] Master 127.16.60.125:32949 requested a full tablet report, sending...
09:15:18.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.210459 16755 ts_manager.cc:194] Registered new tserver with Master: ac5023c640ae4ccdb6e27ef5cfb9c524 (127.16.60.66:34795)
09:15:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:18.508112 17258 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:15:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:18.508572 17258 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:15:18.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:18.509128 17258 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:15:18.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:18.540759 17258 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:15:18.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:18.541625 17258 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:15:18.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:18.541833 17258 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:15:18.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:18.541983 17258 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:15:18.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:18.542382 17258 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.60.67
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.579011 17258 tablet_server_runner.cc:78] Tablet server non-default flags:
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/data
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/wal
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.67:0
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/data/info.pb
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.67
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.60.67
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/logs
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Jan 2026 08:43:23 UTC on 5fd53c4cbb9d
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10261
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:15:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.580303 17258 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:15:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:18.581959 17258 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:15:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:18.595117 17264 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:19.208674 17253 heartbeater.cc:499] Master 127.16.60.124:44777 was elected leader, sending a full tablet report...
09:15:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:20.000707 17263 debug-util.cc:398] Leaking SignalData structure 0x7b0800037660 after lost signal to thread 17258
09:15:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:20.277933 17263 kernel_stack_watchdog.cc:198] Thread 17258 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
09:15:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:15:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:15:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:15:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:15:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:18.596482 17265 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:20.278676 17258 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.683s user 0.659s sys 0.931s
09:15:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:20.279062 17258 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.684s user 0.659s sys 0.931s
09:15:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:20.279629 17266 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1681 milliseconds
09:15:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.280797 17258 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:15:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:20.280910 17267 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:20.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.284200 17258 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.286473 17258 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:15:20.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.287812 17258 hybrid_clock.cc:648] HybridClock initialized: now 1769764520287784 us; error 54 us; skew 500 ppm
09:15:20.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.288587 17258 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:15:20.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.294914 17258 webserver.cc:492] Webserver started at http://127.16.60.67:36803/ using document root <none> and password file <none>
09:15:20.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.295771 17258 fs_manager.cc:362] Metadata directory not provided
09:15:20.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.296096 17258 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:15:20.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.296558 17258 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
09:15:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.300776 17258 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/data/instance:
09:15:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1eb0155aadb4d13af735b85265f3aed"
09:15:20.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:20 on dist-test-slave-0ffw"
09:15:20.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.301858 17258 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/wal/instance:
09:15:20.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1eb0155aadb4d13af735b85265f3aed"
09:15:20.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:20 on dist-test-slave-0ffw"
09:15:20.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.308670 17258 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
09:15:20.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.314424 17276 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:15:20.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.315510 17258 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.315804 17258 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/data,/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/wal
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1eb0155aadb4d13af735b85265f3aed"
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:20 on dist-test-slave-0ffw"
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.316083 17258 fs_report.cc:389] FS layout report
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/wal
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/wal
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/data/data
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:15:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.369709 17258 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:15:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.371203 17258 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:15:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.371610 17258 kserver.cc:163] Server-wide thread pool size limit: 3276
09:15:20.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.382354 17258 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:15:20.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.382616 17258 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:15:20.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.382946 17258 ts_tablet_manager.cc:616] Registered 0 tablets
09:15:20.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.383136 17258 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:15:20.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.571798 17258 rpc_server.cc:307] RPC server started. Bound to: 127.16.60.67:46487
09:15:20.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.572245 17400 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.60.67:46487 every 8 connection(s)
09:15:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.574451 17258 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/data/info.pb
09:15:20.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.579371 16625 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu as pid 17258
09:15:20.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.579845 16625 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/ts-2/wal/instance
09:15:20.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.590193 17403 heartbeater.cc:344] Connected to a master server at 127.16.60.126:39927
09:15:20.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.590639 17403 heartbeater.cc:461] Registering TS with master...
09:15:20.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.592407 17403 heartbeater.cc:507] Master 127.16.60.126:39927 requested a full tablet report, sending...
09:15:20.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.595350 16670 ts_manager.cc:194] Registered new tserver with Master: a1eb0155aadb4d13af735b85265f3aed (127.16.60.67:46487)
09:15:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.600488 17402 heartbeater.cc:344] Connected to a master server at 127.16.60.125:32949
09:15:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.601392 17402 heartbeater.cc:461] Registering TS with master...
09:15:20.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.602286 17402 heartbeater.cc:507] Master 127.16.60.125:32949 requested a full tablet report, sending...
09:15:20.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.603525 17401 heartbeater.cc:344] Connected to a master server at 127.16.60.124:44777
09:15:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.603850 17401 heartbeater.cc:461] Registering TS with master...
09:15:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.604692 17401 heartbeater.cc:507] Master 127.16.60.124:44777 requested a full tablet report, sending...
09:15:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.604812 16755 ts_manager.cc:194] Registered new tserver with Master: a1eb0155aadb4d13af735b85265f3aed (127.16.60.67:46487)
09:15:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.606683 16846 ts_manager.cc:194] Registered new tserver with Master: a1eb0155aadb4d13af735b85265f3aed (127.16.60.67:46487)
09:15:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.608405 16846 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.60.67:42229
09:15:20.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.620440 16625 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
09:15:20.631 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:15:20.631 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
09:15:20.632 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.16.60.126"
port: 39927
}
bound_http_address {
host: "127.16.60.126"
port: 40961
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.16.60.125"
port: 32949
}
bound_http_address {
host: "127.16.60.125"
port: 42103
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.16.60.124"
port: 44777
}
bound_http_address {
host: "127.16.60.124"
port: 42633
}
}
}
09:15:20.632 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:15:20.633 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.16.60.65"
port: 35089
}
bound_http_address {
host: "127.16.60.65"
port: 37537
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.16.60.66"
port: 34795
}
bound_http_address {
host: "127.16.60.66"
port: 42831
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.16.60.67"
port: 46487
}
bound_http_address {
host: "127.16.60.67"
port: 36803
}
}
}
09:15:20.633 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
09:15:20.635 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
09:15:20.637 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.60.125:32949(127.16.60.125:32949)] connecting to peer
09:15:20.638 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:20.639 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.60.125:32949(127.16.60.125:32949)] Successfully connected to peer
09:15:20.639 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.60.124:44777(127.16.60.124:44777)] connecting to peer
09:15:20.639 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.16.60.126:39927(127.16.60.126:39927)] Successfully connected to peer
09:15:20.640 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.60.124:44777(127.16.60.124:44777)] Successfully connected to peer
09:15:20.641 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.60.125
09:15:20.643 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.60.126
09:15:20.645 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.60.124
09:15:20.661 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x15cfc92c, L:/127.0.0.1:58268 - R:/127.16.60.125:32949] using SASL/PLAIN
09:15:20.661 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x441b591c, L:/127.0.0.1:34908 - R:/127.16.60.126:39927] using SASL/PLAIN
09:15:20.662 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x1d4506ad, L:/127.0.0.1:54780 - R:/127.16.60.124:44777] using SASL/PLAIN
09:15:20.666 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:15:20.667 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=30), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [27ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [27ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: OK, [29ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, deferred=null) after lookup
09:15:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.669399 16846 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54780:
09:15:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
09:15:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:15:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:15:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:15:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:20.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:20.673662 16846 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:15:20.708 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 8443fe4bd22a423ba4cb25bc662d8a4a
09:15:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.766777 17188 tablet_service.cc:1505] Processing CreateTablet for tablet 49164ca60c4f4a87b9cdf907d2afea7a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:15:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.767719 17187 tablet_service.cc:1505] Processing CreateTablet for tablet 2d3db7ed5f984b45b8fe995cef9d6d8e (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:15:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.768838 17188 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 49164ca60c4f4a87b9cdf907d2afea7a. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.769761 17187 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2d3db7ed5f984b45b8fe995cef9d6d8e. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.778013 17038 tablet_service.cc:1505] Processing CreateTablet for tablet 2d3db7ed5f984b45b8fe995cef9d6d8e (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:15:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.779979 17038 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2d3db7ed5f984b45b8fe995cef9d6d8e. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:20.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.783753 17039 tablet_service.cc:1505] Processing CreateTablet for tablet 49164ca60c4f4a87b9cdf907d2afea7a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:15:20.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.785049 17039 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 49164ca60c4f4a87b9cdf907d2afea7a. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:20.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.785655 17335 tablet_service.cc:1505] Processing CreateTablet for tablet 2d3db7ed5f984b45b8fe995cef9d6d8e (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:15:20.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.787710 17335 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2d3db7ed5f984b45b8fe995cef9d6d8e. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.789961 17336 tablet_service.cc:1505] Processing CreateTablet for tablet 49164ca60c4f4a87b9cdf907d2afea7a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:15:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.791406 17336 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 49164ca60c4f4a87b9cdf907d2afea7a. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.808342 17427 tablet_bootstrap.cc:492] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524: Bootstrap starting.
09:15:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.815568 17427 tablet_bootstrap.cc:654] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524: Neither blocks nor log segments found. Creating new log.
09:15:20.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.817966 17427 log.cc:826] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524: Log is configured to *not* fsync() on all Append() calls
09:15:20.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.829074 17428 tablet_bootstrap.cc:492] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa: Bootstrap starting.
09:15:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.833117 17427 tablet_bootstrap.cc:492] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524: No bootstrap required, opened a new log
09:15:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.833696 17427 ts_tablet_manager.cc:1403] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524: Time spent bootstrapping tablet: real 0.026s user 0.014s sys 0.010s
09:15:20.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.836606 17428 tablet_bootstrap.cc:654] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa: Neither blocks nor log segments found. Creating new log.
09:15:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.838996 17428 log.cc:826] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa: Log is configured to *not* fsync() on all Append() calls
09:15:20.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.852298 17428 tablet_bootstrap.cc:492] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa: No bootstrap required, opened a new log
09:15:20.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.852818 17428 ts_tablet_manager.cc:1403] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa: Time spent bootstrapping tablet: real 0.025s user 0.014s sys 0.004s
09:15:20.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.854049 17431 tablet_bootstrap.cc:492] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed: Bootstrap starting.
09:15:20.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.859974 17427 raft_consensus.cc:359] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.860913 17427 raft_consensus.cc:385] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.861202 17427 raft_consensus.cc:740] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ac5023c640ae4ccdb6e27ef5cfb9c524, State: Initialized, Role: FOLLOWER
09:15:20.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.862334 17431 tablet_bootstrap.cc:654] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed: Neither blocks nor log segments found. Creating new log.
09:15:20.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.862111 17427 consensus_queue.cc:260] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.864660 17431 log.cc:826] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed: Log is configured to *not* fsync() on all Append() calls
09:15:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.865933 17427 ts_tablet_manager.cc:1434] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524: Time spent starting tablet: real 0.032s user 0.030s sys 0.002s
09:15:20.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.876137 17431 tablet_bootstrap.cc:492] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed: No bootstrap required, opened a new log
09:15:20.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.876451 17427 tablet_bootstrap.cc:492] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524: Bootstrap starting.
09:15:20.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.876662 17431 ts_tablet_manager.cc:1403] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed: Time spent bootstrapping tablet: real 0.023s user 0.016s sys 0.004s
09:15:20.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.882614 17427 tablet_bootstrap.cc:654] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524: Neither blocks nor log segments found. Creating new log.
09:15:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.882570 17428 raft_consensus.cc:359] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.883469 17428 raft_consensus.cc:385] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:20.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.883791 17428 raft_consensus.cc:740] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1aefced973a34ed69a4c40fe377b10aa, State: Initialized, Role: FOLLOWER
09:15:20.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.884778 17428 consensus_queue.cc:260] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.895026 17428 ts_tablet_manager.cc:1434] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa: Time spent starting tablet: real 0.042s user 0.029s sys 0.012s
09:15:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.896114 17428 tablet_bootstrap.cc:492] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa: Bootstrap starting.
09:15:20.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.906306 17431 raft_consensus.cc:359] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.907419 17428 tablet_bootstrap.cc:654] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa: Neither blocks nor log segments found. Creating new log.
09:15:20.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.907343 17431 raft_consensus.cc:385] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:20.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.907663 17431 raft_consensus.cc:740] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a1eb0155aadb4d13af735b85265f3aed, State: Initialized, Role: FOLLOWER
09:15:20.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.908722 17431 consensus_queue.cc:260] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.911561 17427 tablet_bootstrap.cc:492] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524: No bootstrap required, opened a new log
09:15:20.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.912252 17427 ts_tablet_manager.cc:1403] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524: Time spent bootstrapping tablet: real 0.036s user 0.013s sys 0.012s
09:15:20.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.914573 17431 ts_tablet_manager.cc:1434] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed: Time spent starting tablet: real 0.038s user 0.021s sys 0.016s
09:15:20.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.915488 17401 heartbeater.cc:499] Master 127.16.60.124:44777 was elected leader, sending a full tablet report...
09:15:20.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.915542 17427 raft_consensus.cc:359] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.916226 17427 raft_consensus.cc:385] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:20.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.916503 17427 raft_consensus.cc:740] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ac5023c640ae4ccdb6e27ef5cfb9c524, State: Initialized, Role: FOLLOWER
09:15:20.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.918567 17431 tablet_bootstrap.cc:492] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed: Bootstrap starting.
09:15:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.917157 17427 consensus_queue.cc:260] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.921229 17427 ts_tablet_manager.cc:1434] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524: Time spent starting tablet: real 0.008s user 0.002s sys 0.003s
09:15:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.928094 17431 tablet_bootstrap.cc:654] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed: Neither blocks nor log segments found. Creating new log.
09:15:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.945070 17431 tablet_bootstrap.cc:492] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed: No bootstrap required, opened a new log
09:15:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.945536 17431 ts_tablet_manager.cc:1403] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed: Time spent bootstrapping tablet: real 0.027s user 0.013s sys 0.007s
09:15:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.945724 17428 tablet_bootstrap.cc:492] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa: No bootstrap required, opened a new log
09:15:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.946164 17428 ts_tablet_manager.cc:1403] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa: Time spent bootstrapping tablet: real 0.050s user 0.018s sys 0.015s
09:15:20.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.948113 17431 raft_consensus.cc:359] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.948802 17431 raft_consensus.cc:385] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.949069 17431 raft_consensus.cc:740] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a1eb0155aadb4d13af735b85265f3aed, State: Initialized, Role: FOLLOWER
09:15:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.948720 17428 raft_consensus.cc:359] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.949445 17428 raft_consensus.cc:385] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.949712 17428 raft_consensus.cc:740] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1aefced973a34ed69a4c40fe377b10aa, State: Initialized, Role: FOLLOWER
09:15:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.949846 17431 consensus_queue.cc:260] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.950359 17428 consensus_queue.cc:260] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.952845 17428 ts_tablet_manager.cc:1434] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa: Time spent starting tablet: real 0.006s user 0.006s sys 0.000s
09:15:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:20.953512 17257 tablet.cc:2378] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:15:20.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.956641 17431 ts_tablet_manager.cc:1434] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed: Time spent starting tablet: real 0.011s user 0.006s sys 0.000s
09:15:20.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.971201 17434 raft_consensus.cc:493] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:15:20.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.971745 17434 raft_consensus.cc:515] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.974386 17434 leader_election.cc:290] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ac5023c640ae4ccdb6e27ef5cfb9c524 (127.16.60.66:34795), a1eb0155aadb4d13af735b85265f3aed (127.16.60.67:46487)
09:15:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.987761 17356 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2d3db7ed5f984b45b8fe995cef9d6d8e" candidate_uuid: "1aefced973a34ed69a4c40fe377b10aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1eb0155aadb4d13af735b85265f3aed" is_pre_election: true
09:15:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.987890 17208 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2d3db7ed5f984b45b8fe995cef9d6d8e" candidate_uuid: "1aefced973a34ed69a4c40fe377b10aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" is_pre_election: true
09:15:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.988538 17356 raft_consensus.cc:2468] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1aefced973a34ed69a4c40fe377b10aa in term 0.
09:15:20.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.988613 17208 raft_consensus.cc:2468] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1aefced973a34ed69a4c40fe377b10aa in term 0.
09:15:20.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.989810 16980 leader_election.cc:304] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1aefced973a34ed69a4c40fe377b10aa, a1eb0155aadb4d13af735b85265f3aed; no voters:
09:15:20.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.990635 17434 raft_consensus.cc:2804] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Leader pre-election won for term 1
09:15:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.990993 17434 raft_consensus.cc:493] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:15:20.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.991333 17434 raft_consensus.cc:3060] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Advancing to term 1
09:15:20.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.995816 17434 raft_consensus.cc:515] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:20.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.997573 17434 leader_election.cc:290] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [CANDIDATE]: Term 1 election: Requested vote from peers ac5023c640ae4ccdb6e27ef5cfb9c524 (127.16.60.66:34795), a1eb0155aadb4d13af735b85265f3aed (127.16.60.67:46487)
09:15:20.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.998385 17208 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2d3db7ed5f984b45b8fe995cef9d6d8e" candidate_uuid: "1aefced973a34ed69a4c40fe377b10aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524"
09:15:20.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.998660 17356 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2d3db7ed5f984b45b8fe995cef9d6d8e" candidate_uuid: "1aefced973a34ed69a4c40fe377b10aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1eb0155aadb4d13af735b85265f3aed"
09:15:20.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.998940 17208 raft_consensus.cc:3060] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Advancing to term 1
09:15:20.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:20.999169 17356 raft_consensus.cc:3060] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Advancing to term 1
09:15:21.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.005972 17356 raft_consensus.cc:2468] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1aefced973a34ed69a4c40fe377b10aa in term 1.
09:15:21.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.005972 17208 raft_consensus.cc:2468] T 2d3db7ed5f984b45b8fe995cef9d6d8e P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1aefced973a34ed69a4c40fe377b10aa in term 1.
09:15:21.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.006937 16980 leader_election.cc:304] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1aefced973a34ed69a4c40fe377b10aa, a1eb0155aadb4d13af735b85265f3aed; no voters:
09:15:21.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.007578 17434 raft_consensus.cc:2804] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [term 1 FOLLOWER]: Leader election won for term 1
09:15:21.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.009447 17434 raft_consensus.cc:697] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [term 1 LEADER]: Becoming Leader. State: Replica: 1aefced973a34ed69a4c40fe377b10aa, State: Running, Role: LEADER
09:15:21.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:21.010704 17108 tablet.cc:2378] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:15:21.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.010474 17434 consensus_queue.cc:237] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.022253 16846 catalog_manager.cc:5654] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 1aefced973a34ed69a4c40fe377b10aa (127.16.60.65). New cstate: current_term: 1 leader_uuid: "1aefced973a34ed69a4c40fe377b10aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } health_report { overall_health: UNKNOWN } } }
09:15:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.071182 17434 raft_consensus.cc:493] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:15:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.071558 17434 raft_consensus.cc:515] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.073240 17434 leader_election.cc:290] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ac5023c640ae4ccdb6e27ef5cfb9c524 (127.16.60.66:34795), a1eb0155aadb4d13af735b85265f3aed (127.16.60.67:46487)
09:15:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.073839 17208 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "49164ca60c4f4a87b9cdf907d2afea7a" candidate_uuid: "1aefced973a34ed69a4c40fe377b10aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" is_pre_election: true
09:15:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.074321 17356 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "49164ca60c4f4a87b9cdf907d2afea7a" candidate_uuid: "1aefced973a34ed69a4c40fe377b10aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1eb0155aadb4d13af735b85265f3aed" is_pre_election: true
09:15:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.074473 17208 raft_consensus.cc:2468] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1aefced973a34ed69a4c40fe377b10aa in term 0.
09:15:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.074771 17356 raft_consensus.cc:2468] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1aefced973a34ed69a4c40fe377b10aa in term 0.
09:15:21.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.075584 16979 leader_election.cc:304] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1aefced973a34ed69a4c40fe377b10aa, ac5023c640ae4ccdb6e27ef5cfb9c524; no voters:
09:15:21.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.076361 17434 raft_consensus.cc:2804] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Leader pre-election won for term 1
09:15:21.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.076681 17434 raft_consensus.cc:493] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:15:21.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.076894 17434 raft_consensus.cc:3060] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Advancing to term 1
09:15:21.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.081689 17434 raft_consensus.cc:515] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.083000 17434 leader_election.cc:290] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [CANDIDATE]: Term 1 election: Requested vote from peers ac5023c640ae4ccdb6e27ef5cfb9c524 (127.16.60.66:34795), a1eb0155aadb4d13af735b85265f3aed (127.16.60.67:46487)
09:15:21.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.083765 17208 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "49164ca60c4f4a87b9cdf907d2afea7a" candidate_uuid: "1aefced973a34ed69a4c40fe377b10aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524"
09:15:21.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.083940 17356 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "49164ca60c4f4a87b9cdf907d2afea7a" candidate_uuid: "1aefced973a34ed69a4c40fe377b10aa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1eb0155aadb4d13af735b85265f3aed"
09:15:21.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.084295 17208 raft_consensus.cc:3060] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Advancing to term 1
09:15:21.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.084441 17356 raft_consensus.cc:3060] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Advancing to term 1
09:15:21.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.090759 17208 raft_consensus.cc:2468] T 49164ca60c4f4a87b9cdf907d2afea7a P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1aefced973a34ed69a4c40fe377b10aa in term 1.
09:15:21.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.090759 17356 raft_consensus.cc:2468] T 49164ca60c4f4a87b9cdf907d2afea7a P a1eb0155aadb4d13af735b85265f3aed [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1aefced973a34ed69a4c40fe377b10aa in term 1.
09:15:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.091723 16979 leader_election.cc:304] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1aefced973a34ed69a4c40fe377b10aa, ac5023c640ae4ccdb6e27ef5cfb9c524; no voters:
09:15:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.092334 17434 raft_consensus.cc:2804] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [term 1 FOLLOWER]: Leader election won for term 1
09:15:21.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.092733 17434 raft_consensus.cc:697] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [term 1 LEADER]: Becoming Leader. State: Replica: 1aefced973a34ed69a4c40fe377b10aa, State: Running, Role: LEADER
09:15:21.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.093485 17434 consensus_queue.cc:237] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:21.096801 17405 tablet.cc:2378] T 2d3db7ed5f984b45b8fe995cef9d6d8e P a1eb0155aadb4d13af735b85265f3aed: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:15:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.102213 16846 catalog_manager.cc:5654] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa reported cstate change: term changed from 0 to 1, leader changed from <none> to 1aefced973a34ed69a4c40fe377b10aa (127.16.60.65). New cstate: current_term: 1 leader_uuid: "1aefced973a34ed69a4c40fe377b10aa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } health_report { overall_health: UNKNOWN } } }
09:15:21.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.462289 17434 consensus_queue.cc:1048] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:15:21.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.479681 17434 consensus_queue.cc:1048] T 2d3db7ed5f984b45b8fe995cef9d6d8e P 1aefced973a34ed69a4c40fe377b10aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:15:21.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.622530 17434 consensus_queue.cc:1048] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:15:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.636031 17434 consensus_queue.cc:1048] T 49164ca60c4f4a87b9cdf907d2afea7a P 1aefced973a34ed69a4c40fe377b10aa [LEADER]: Connected to new peer: Peer: permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.791590 16846 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54786:
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:15:21.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:21.794160 16846 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:15:21.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.831399 17038 tablet_service.cc:1505] Processing CreateTablet for tablet 6f5013700e894c0981651f54fb02c2b9 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=f4146e3da09f408794f6caec5133a28a]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:15:21.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.833148 17038 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6f5013700e894c0981651f54fb02c2b9. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.833396 17187 tablet_service.cc:1505] Processing CreateTablet for tablet 6f5013700e894c0981651f54fb02c2b9 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=f4146e3da09f408794f6caec5133a28a]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:15:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.834970 17187 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6f5013700e894c0981651f54fb02c2b9. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.834676 17336 tablet_service.cc:1505] Processing CreateTablet for tablet 6f5013700e894c0981651f54fb02c2b9 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=f4146e3da09f408794f6caec5133a28a]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:15:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.836195 17336 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6f5013700e894c0981651f54fb02c2b9. 1 dirs total, 0 dirs full, 0 dirs failed
09:15:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.850450 17473 tablet_bootstrap.cc:492] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa: Bootstrap starting.
09:15:21.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.855798 17475 tablet_bootstrap.cc:492] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed: Bootstrap starting.
09:15:21.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.857224 17474 tablet_bootstrap.cc:492] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: Bootstrap starting.
09:15:21.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.857904 17473 tablet_bootstrap.cc:654] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa: Neither blocks nor log segments found. Creating new log.
09:15:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.861954 17475 tablet_bootstrap.cc:654] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed: Neither blocks nor log segments found. Creating new log.
09:15:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.862383 17474 tablet_bootstrap.cc:654] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: Neither blocks nor log segments found. Creating new log.
09:15:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.864410 17473 tablet_bootstrap.cc:492] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa: No bootstrap required, opened a new log
09:15:21.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.864822 17473 ts_tablet_manager.cc:1403] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa: Time spent bootstrapping tablet: real 0.015s user 0.004s sys 0.005s
09:15:21.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.867578 17475 tablet_bootstrap.cc:492] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed: No bootstrap required, opened a new log
09:15:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.867978 17475 ts_tablet_manager.cc:1403] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed: Time spent bootstrapping tablet: real 0.012s user 0.007s sys 0.002s
09:15:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.867565 17473 raft_consensus.cc:359] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.868216 17473 raft_consensus.cc:385] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.868618 17474 tablet_bootstrap.cc:492] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: No bootstrap required, opened a new log
09:15:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.868506 17473 raft_consensus.cc:740] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1aefced973a34ed69a4c40fe377b10aa, State: Initialized, Role: FOLLOWER
09:15:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.869170 17474 ts_tablet_manager.cc:1403] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: Time spent bootstrapping tablet: real 0.012s user 0.010s sys 0.000s
09:15:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.870100 17473 consensus_queue.cc:260] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.871479 17434 tablet_replica.cc:442] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } } }
09:15:21.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.872161 17434 tablet_replica.cc:445] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa: This TxnStatusTablet replica's current role is: FOLLOWER
09:15:21.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.871886 17474 raft_consensus.cc:359] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.872547 17474 raft_consensus.cc:385] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:21.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.872813 17474 raft_consensus.cc:740] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ac5023c640ae4ccdb6e27ef5cfb9c524, State: Initialized, Role: FOLLOWER
09:15:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.873564 17474 consensus_queue.cc:260] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.875285 17473 ts_tablet_manager.cc:1434] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa: Time spent starting tablet: real 0.010s user 0.004s sys 0.004s
09:15:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.874974 17449 tablet_replica.cc:442] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } } }
09:15:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.875682 17449 tablet_replica.cc:445] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: This TxnStatusTablet replica's current role is: FOLLOWER
09:15:21.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.883519 17474 ts_tablet_manager.cc:1434] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: Time spent starting tablet: real 0.014s user 0.004s sys 0.001s
09:15:21.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.870807 17475 raft_consensus.cc:359] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.889991 17475 raft_consensus.cc:385] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:15:21.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.890357 17475 raft_consensus.cc:740] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a1eb0155aadb4d13af735b85265f3aed, State: Initialized, Role: FOLLOWER
09:15:21.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.891013 17475 consensus_queue.cc:260] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.892243 17444 tablet_replica.cc:442] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } } }
09:15:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.892966 17444 tablet_replica.cc:445] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed: This TxnStatusTablet replica's current role is: FOLLOWER
09:15:21.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.898852 17475 ts_tablet_manager.cc:1434] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed: Time spent starting tablet: real 0.031s user 0.005s sys 0.002s
09:15:21.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.921741 17449 raft_consensus.cc:493] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:15:21.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.922132 17449 raft_consensus.cc:515] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.924664 17449 leader_election.cc:290] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1aefced973a34ed69a4c40fe377b10aa (127.16.60.65:35089), a1eb0155aadb4d13af735b85265f3aed (127.16.60.67:46487)
09:15:21.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.936173 17059 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "6f5013700e894c0981651f54fb02c2b9" candidate_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1aefced973a34ed69a4c40fe377b10aa" is_pre_election: true
09:15:21.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.936926 17059 raft_consensus.cc:2468] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ac5023c640ae4ccdb6e27ef5cfb9c524 in term 0.
09:15:21.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.938366 17130 leader_election.cc:304] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1aefced973a34ed69a4c40fe377b10aa, ac5023c640ae4ccdb6e27ef5cfb9c524; no voters:
09:15:21.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.939286 17449 raft_consensus.cc:2804] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:15:21.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.939580 17449 raft_consensus.cc:493] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:15:21.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.939786 17449 raft_consensus.cc:3060] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 0 FOLLOWER]: Advancing to term 1
09:15:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.940662 17356 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "6f5013700e894c0981651f54fb02c2b9" candidate_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1eb0155aadb4d13af735b85265f3aed" is_pre_election: true
09:15:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.941283 17356 raft_consensus.cc:2468] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ac5023c640ae4ccdb6e27ef5cfb9c524 in term 0.
09:15:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.944880 17449 raft_consensus.cc:515] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.946249 17449 leader_election.cc:290] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [CANDIDATE]: Term 1 election: Requested vote from peers 1aefced973a34ed69a4c40fe377b10aa (127.16.60.65:35089), a1eb0155aadb4d13af735b85265f3aed (127.16.60.67:46487)
09:15:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.946995 17059 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "6f5013700e894c0981651f54fb02c2b9" candidate_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1aefced973a34ed69a4c40fe377b10aa"
09:15:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.947176 17356 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "6f5013700e894c0981651f54fb02c2b9" candidate_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1eb0155aadb4d13af735b85265f3aed"
09:15:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.947383 17059 raft_consensus.cc:3060] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa [term 0 FOLLOWER]: Advancing to term 1
09:15:21.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.947698 17356 raft_consensus.cc:3060] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed [term 0 FOLLOWER]: Advancing to term 1
09:15:21.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.953122 17059 raft_consensus.cc:2468] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ac5023c640ae4ccdb6e27ef5cfb9c524 in term 1.
09:15:21.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.953887 17356 raft_consensus.cc:2468] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ac5023c640ae4ccdb6e27ef5cfb9c524 in term 1.
09:15:21.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.954111 17130 leader_election.cc:304] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1aefced973a34ed69a4c40fe377b10aa, ac5023c640ae4ccdb6e27ef5cfb9c524; no voters:
09:15:21.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.954885 17449 raft_consensus.cc:2804] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 1 FOLLOWER]: Leader election won for term 1
09:15:21.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.956665 17449 raft_consensus.cc:697] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [term 1 LEADER]: Becoming Leader. State: Replica: ac5023c640ae4ccdb6e27ef5cfb9c524, State: Running, Role: LEADER
09:15:21.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.957639 17449 consensus_queue.cc:237] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } }
09:15:21.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.968981 17483 tablet_replica.cc:442] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: TxnStatusTablet state changed. Reason: New leader ac5023c640ae4ccdb6e27ef5cfb9c524. Latest consensus state: current_term: 1 leader_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } } }
09:15:21.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.969772 17483 tablet_replica.cc:445] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: This TxnStatusTablet replica's current role is: LEADER
09:15:21.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.971944 17485 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
09:15:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.972851 17485 txn_status_manager.cc:930] Loading transaction status metadata into memory...
09:15:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:21.973338 16846 catalog_manager.cc:5654] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 reported cstate change: term changed from 0 to 1, leader changed from <none> to ac5023c640ae4ccdb6e27ef5cfb9c524 (127.16.60.66). New cstate: current_term: 1 leader_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } health_report { overall_health: UNKNOWN } } }
09:15:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.032939 17356 raft_consensus.cc:1275] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed [term 1 FOLLOWER]: Refusing update from remote peer ac5023c640ae4ccdb6e27ef5cfb9c524: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:15:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.034348 17059 raft_consensus.cc:1275] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa [term 1 FOLLOWER]: Refusing update from remote peer ac5023c640ae4ccdb6e27ef5cfb9c524: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:15:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.035390 17483 consensus_queue.cc:1048] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:15:22.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.037003 17449 consensus_queue.cc:1048] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:15:22.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.055194 17444 tablet_replica.cc:442] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed: TxnStatusTablet state changed. Reason: New leader ac5023c640ae4ccdb6e27ef5cfb9c524. Latest consensus state: current_term: 1 leader_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } } }
09:15:22.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.055855 17444 tablet_replica.cc:445] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed: This TxnStatusTablet replica's current role is: FOLLOWER
09:15:22.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.061847 17451 mvcc.cc:204] Tried to move back new op lower bound from 7248955482231631872 to 7248955481949851648. Current Snapshot: MvccSnapshot[applied={T|T < 7248955482231631872}]
09:15:22.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.060999 17441 tablet_replica.cc:442] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa: TxnStatusTablet state changed. Reason: New leader ac5023c640ae4ccdb6e27ef5cfb9c524. Latest consensus state: current_term: 1 leader_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } } }
09:15:22.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.073401 17441 tablet_replica.cc:445] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa: This TxnStatusTablet replica's current role is: FOLLOWER
09:15:22.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.074928 17449 tablet_replica.cc:442] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } } }
09:15:22.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.075593 17449 tablet_replica.cc:445] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: This TxnStatusTablet replica's current role is: LEADER
09:15:22.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.076505 17449 tablet_replica.cc:442] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } } }
09:15:22.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.077097 17449 tablet_replica.cc:445] T 6f5013700e894c0981651f54fb02c2b9 P ac5023c640ae4ccdb6e27ef5cfb9c524: This TxnStatusTablet replica's current role is: LEADER
09:15:22.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.078722 17447 tablet_replica.cc:442] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } } }
09:15:22.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.079460 17447 tablet_replica.cc:445] T 6f5013700e894c0981651f54fb02c2b9 P 1aefced973a34ed69a4c40fe377b10aa: This TxnStatusTablet replica's current role is: FOLLOWER
09:15:22.100 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
09:15:22.104 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 8443fe4bd22a423ba4cb25bc662d8a4a
09:15:22.106 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.60.124:44777
09:15:22.107 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
09:15:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.107672 16625 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu with pid 16802
09:15:22.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.115784 17444 tablet_replica.cc:442] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" member_type: VOTER last_known_addr { host: "127.16.60.65" port: 35089 } } peers { permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" member_type: VOTER last_known_addr { host: "127.16.60.66" port: 34795 } } peers { permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" member_type: VOTER last_known_addr { host: "127.16.60.67" port: 46487 } } }
09:15:22.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.116461 17444 tablet_replica.cc:445] T 6f5013700e894c0981651f54fb02c2b9 P a1eb0155aadb4d13af735b85265f3aed: This TxnStatusTablet replica's current role is: FOLLOWER
09:15:22.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:22.144001 17280 connection.cc:537] client connection to 127.16.60.124:44777 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:15:22.145 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.60.124:44777(127.16.60.124:44777)] handling channelInactive
09:15:22.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.124:44777(127.16.60.124:44777)] cleaning up while in state READY due to: connection closed
09:15:22.146 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:15:22.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:22.146775 17401 heartbeater.cc:646] Failed to heartbeat to 127.16.60.124:44777 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:15:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.369071 17494 raft_consensus.cc:493] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c796ccb9931245c8a473b1b23e405086)
09:15:22.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.369711 17494 raft_consensus.cc:515] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:22.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.371745 17494 leader_election.cc:290] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949), c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777)
09:15:22.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.372721 16765 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661dc84b98204b03aaa17fef82fe8abc" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" is_pre_election: true
09:15:22.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:22.377995 16657 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111) [suppressed 13 similar messages]
09:15:22.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:22.381565 16657 leader_election.cc:336] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777): Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:22.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.382005 16657 leader_election.cc:304] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 661dc84b98204b03aaa17fef82fe8abc; no voters: c796ccb9931245c8a473b1b23e405086, d9e5dacfdeb94d4d8bf1f17195e777b4
09:15:22.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.382756 17494 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
09:15:22.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.399631 17497 raft_consensus.cc:493] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c796ccb9931245c8a473b1b23e405086)
09:15:22.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.400099 17497 raft_consensus.cc:515] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:22.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.402253 17497 leader_election.cc:290] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927), c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777)
09:15:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.403132 16680 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "661dc84b98204b03aaa17fef82fe8abc" is_pre_election: true
09:15:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.403735 16680 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d9e5dacfdeb94d4d8bf1f17195e777b4 in term 1.
09:15:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:22.404940 16742 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111) [suppressed 6 similar messages]
09:15:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.404865 16741 leader_election.cc:304] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 661dc84b98204b03aaa17fef82fe8abc, d9e5dacfdeb94d4d8bf1f17195e777b4; no voters:
09:15:22.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.405750 17497 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 1 FOLLOWER]: Leader pre-election won for term 2
09:15:22.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.406085 17497 raft_consensus.cc:493] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 1 FOLLOWER]: Starting leader election (detected failure of leader c796ccb9931245c8a473b1b23e405086)
09:15:22.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.406410 17497 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 1 FOLLOWER]: Advancing to term 2
09:15:22.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:22.408697 16742 leader_election.cc:336] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777): Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:22.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.414700 17497 raft_consensus.cc:515] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:22.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.416509 17497 leader_election.cc:290] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [CANDIDATE]: Term 2 election: Requested vote from peers 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927), c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777)
09:15:22.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.417260 16680 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "661dc84b98204b03aaa17fef82fe8abc"
09:15:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.417825 16680 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 1 FOLLOWER]: Advancing to term 2
09:15:22.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:22.420186 16742 leader_election.cc:336] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777): Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.423390 16680 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d9e5dacfdeb94d4d8bf1f17195e777b4 in term 2.
09:15:22.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.424201 16741 leader_election.cc:304] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 661dc84b98204b03aaa17fef82fe8abc, d9e5dacfdeb94d4d8bf1f17195e777b4; no voters: c796ccb9931245c8a473b1b23e405086
09:15:22.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.424813 17497 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 2 FOLLOWER]: Leader election won for term 2
09:15:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.426576 17497 raft_consensus.cc:697] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 2 LEADER]: Becoming Leader. State: Replica: d9e5dacfdeb94d4d8bf1f17195e777b4, State: Running, Role: LEADER
09:15:22.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.427558 17497 consensus_queue.cc:237] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.431264 17500 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9e5dacfdeb94d4d8bf1f17195e777b4. Latest consensus state: current_term: 2 leader_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.431854 17500 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: This master's current role is: LEADER
09:15:22.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.434391 17501 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:15:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.441282 17501 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a]
09:15:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.443118 17501 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=f4146e3da09f408794f6caec5133a28a]
09:15:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.448074 17501 tablet_loader.cc:96] loaded metadata for tablet 2d3db7ed5f984b45b8fe995cef9d6d8e (table txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a])
09:15:22.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.449103 17501 tablet_loader.cc:96] loaded metadata for tablet 49164ca60c4f4a87b9cdf907d2afea7a (table txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a])
09:15:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.450088 17501 tablet_loader.cc:96] loaded metadata for tablet 6f5013700e894c0981651f54fb02c2b9 (table kudu_system.kudu_transactions [id=f4146e3da09f408794f6caec5133a28a])
09:15:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.450934 17501 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:15:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.453869 17501 catalog_manager.cc:1269] Loaded cluster ID: 91258d94f6904e95815ac3730f414487
09:15:22.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.454078 17501 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:15:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.457264 17501 catalog_manager.cc:1514] Loading token signing keys...
09:15:22.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.461184 17501 catalog_manager.cc:6038] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: Loaded TSK: 0
09:15:22.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.462508 17501 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:15:22.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.519102 16680 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 2 FOLLOWER]: Refusing update from remote peer d9e5dacfdeb94d4d8bf1f17195e777b4: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
09:15:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.520469 17500 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
09:15:22.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.523878 17494 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9e5dacfdeb94d4d8bf1f17195e777b4. Latest consensus state: current_term: 2 leader_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:22.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.524614 17494 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: This master's current role is: FOLLOWER
09:15:22.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.528927 17497 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:22.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.529850 17497 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: This master's current role is: LEADER
09:15:22.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:22.532903 16742 consensus_peers.cc:597] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 -> Peer c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777): Couldn't send request to peer c796ccb9931245c8a473b1b23e405086. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:15:22.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.532855 17494 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:22.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.533529 17494 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: This master's current role is: FOLLOWER
09:15:22.618 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.124:44777(127.16.60.124:44777)] connecting to peer
09:15:22.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.60.124:44777(127.16.60.124:44777): Connection refused: /127.16.60.124:44777
09:15:22.620 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.124:44777(127.16.60.124:44777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.124:44777(127.16.60.124:44777): Connection refused: /127.16.60.124:44777
09:15:22.620 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.16.60.124:44777(127.16.60.124:44777) for tablet Kudu Master: Failed to connect to peer master-127.16.60.124:44777(127.16.60.124:44777): Connection refused: /127.16.60.124:44777
09:15:22.638 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.124:44777(127.16.60.124:44777)] connecting to peer
09:15:22.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.60.124:44777(127.16.60.124:44777): Connection refused: /127.16.60.124:44777
09:15:22.644 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.124:44777(127.16.60.124:44777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.124:44777(127.16.60.124:44777): Connection refused: /127.16.60.124:44777
09:15:22.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.124:44777: Failed to connect to peer master-127.16.60.124:44777(127.16.60.124:44777): Connection refused: /127.16.60.124:44777
09:15:22.648 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:15:22.649 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=31), Traces: [0ms] sending RPC to server master-127.16.60.124:44777, [2ms] received response from server master-127.16.60.124:44777: Network error: Failed to connect to peer master-127.16.60.124:44777(127.16.60.124:44777): Connection refused: /127.16.60.124:44777, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.16.60.124:44777(127.16.60.124:44777): Connection refused: /127.16.60.124:44777, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [25ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: OK, [26ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: Network error: Failed to connect to peer master-127.16.60.124:44777(127.16.60.124:44777): Connection refused: /127.16.60.124:44777, [30ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, deferred=Deferred@2032091189(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/280876061@2d616d2b, errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/1864875192@20fc5400)) after lookup
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:22.767191 16754 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53630:
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:15:22.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:15:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:15:22.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:22.770927 16754 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:15:23.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:23.056885 16742 consensus_peers.cc:597] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 -> Peer c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777): Couldn't send request to peer c796ccb9931245c8a473b1b23e405086. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:15:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:23.095880 17253 heartbeater.cc:646] Failed to heartbeat to 127.16.60.124:44777 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:23.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:23.107015 17104 heartbeater.cc:646] Failed to heartbeat to 127.16.60.124:44777 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111)
09:15:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:23.520673 16742 consensus_peers.cc:597] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 -> Peer c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777): Couldn't send request to peer c796ccb9931245c8a473b1b23e405086. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:15:23.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:23.992528 16742 consensus_peers.cc:597] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 -> Peer c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777): Couldn't send request to peer c796ccb9931245c8a473b1b23e405086. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:15:24.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:24.097507 17254 heartbeater.cc:499] Master 127.16.60.125:32949 was elected leader, sending a full tablet report...
09:15:24.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:24.112000 17105 heartbeater.cc:499] Master 127.16.60.125:32949 was elected leader, sending a full tablet report...
09:15:24.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:24.123668 17402 heartbeater.cc:499] Master 127.16.60.125:32949 was elected leader, sending a full tablet report...
09:15:24.154 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 49164ca60c4f4a87b9cdf907d2afea7a for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
09:15:24.155 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 2d3db7ed5f984b45b8fe995cef9d6d8e for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
09:15:24.155 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=49164ca60c4f4a87b9cdf907d2afea7a@[1aefced973a34ed69a4c40fe377b10aa(127.16.60.65:35089)[L],a1eb0155aadb4d13af735b85265f3aed(127.16.60.67:46487),ac5023c640ae4ccdb6e27ef5cfb9c524(127.16.60.66:34795)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=2d3db7ed5f984b45b8fe995cef9d6d8e@[1aefced973a34ed69a4c40fe377b10aa(127.16.60.65:35089)[L],a1eb0155aadb4d13af735b85265f3aed(127.16.60.67:46487),ac5023c640ae4ccdb6e27ef5cfb9c524(127.16.60.66:34795)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
09:15:24.155 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.16.60.125:32949, [3ms] Sub RPC GetTableLocations: received response from server master-127.16.60.125:32949: OK, deferred=null) after lookup
09:15:24.155 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 1aefced973a34ed69a4c40fe377b10aa(127.16.60.65:35089)] connecting to peer
09:15:24.156 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 1aefced973a34ed69a4c40fe377b10aa(127.16.60.65:35089)] Successfully connected to peer
09:15:24.176 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x57a91ca2, L:/127.0.0.1:53088 - R:/127.16.60.65:35089] using TOKEN/null
09:15:24.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.185197 16999 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.16.60.124:44777)
09:15:24.259 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.60.124:44777
09:15:24.259 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:24.261052 16625 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/logs
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data/info.pb
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.124:44777
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.124
09:15:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42633
09:15:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:15:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:15:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
09:15:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:15:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:15:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.497582 16742 consensus_peers.cc:597] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 -> Peer c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777): Couldn't send request to peer c796ccb9931245c8a473b1b23e405086. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
09:15:24.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.571866 17544 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:15:24.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.572388 17544 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:15:24.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.572978 17544 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:15:24.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.608770 17544 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:15:24.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.609138 17544 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:15:24.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.609447 17544 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:15:24.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.609771 17544 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:15:24.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.610131 17544 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:15:24.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.610433 17544 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:15:24.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.610684 17544 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43617/jwks.json
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:24.652604 17544 master_runner.cc:386] Master server non-default flags:
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.124:44777
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
09:15:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data/info.pb
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.124
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42633
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/logs
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Jan 2026 08:43:23 UTC on 5fd53c4cbb9d
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10261
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:15:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:24.654388 17544 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:15:24.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:24.656592 17544 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:15:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.668089 17550 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:24.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.960289 16742 consensus_peers.cc:597] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 -> Peer c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777): Couldn't send request to peer c796ccb9931245c8a473b1b23e405086. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
09:15:25.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:25.521384 16742 consensus_peers.cc:597] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 -> Peer c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777): Couldn't send request to peer c796ccb9931245c8a473b1b23e405086. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
09:15:25.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.672209 17551 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:25.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:24.673834 17544 server_base.cc:1060] running on GCE node
09:15:25.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:24.673198 17553 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:25.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.930022 17544 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:15:25.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.932477 17544 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:15:25.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.933859 17544 hybrid_clock.cc:648] HybridClock initialized: now 1769764525933812 us; error 46 us; skew 500 ppm
09:15:25.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.934705 17544 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:15:25.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.940793 17544 webserver.cc:492] Webserver started at http://127.16.60.124:42633/ using document root <none> and password file <none>
09:15:25.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.941741 17544 fs_manager.cc:362] Metadata directory not provided
09:15:25.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.941937 17544 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:15:25.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.949615 17544 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.007s
09:15:25.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.954846 17564 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:15:25.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.956669 17544 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
09:15:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.957239 17544 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data,/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal
09:15:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c796ccb9931245c8a473b1b23e405086"
09:15:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:11 on dist-test-slave-0ffw"
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:25.959887 17544 fs_report.cc:389] FS layout report
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/wal
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data/data
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:15:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:15:26.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.021838 17544 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:15:26.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.023238 17544 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:15:26.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.023633 17544 kserver.cc:163] Server-wide thread pool size limit: 3276
09:15:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:26.079560 16742 consensus_peers.cc:597] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 -> Peer c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777): Couldn't send request to peer c796ccb9931245c8a473b1b23e405086. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.124:44777: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
09:15:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.128557 17544 rpc_server.cc:307] RPC server started. Bound to: 127.16.60.124:44777
09:15:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.128628 17628 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.60.124:44777 every 8 connection(s)
09:15:26.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.131301 17544 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-2/data/info.pb
09:15:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.138007 16625 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu as pid 17544
09:15:26.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:15:26.139 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.60.125:32949
09:15:26.139 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
09:15:26.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.139470 16625 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu with pid 16717
09:15:26.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.145548 17629 sys_catalog.cc:263] Verifying existing consensus state
09:15:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.151444 17629 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: Bootstrap starting.
09:15:26.181 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.60.125:32949(127.16.60.125:32949)] handling channelInactive
09:15:26.181 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.125:32949(127.16.60.125:32949)] cleaning up while in state READY due to: connection closed
09:15:26.183 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:15:26.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.212893 17401 heartbeater.cc:344] Connected to a master server at 127.16.60.124:44777
09:15:26.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.219877 17629 log.cc:826] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: Log is configured to *not* fsync() on all Append() calls
09:15:26.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.252151 17629 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:15:26.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.253002 17629 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: Bootstrap complete.
09:15:26.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.272642 17629 raft_consensus.cc:359] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:26.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.275067 17629 raft_consensus.cc:740] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: c796ccb9931245c8a473b1b23e405086, State: Initialized, Role: FOLLOWER
09:15:26.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.275887 17629 consensus_queue.cc:260] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:26.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.278213 17634 sys_catalog.cc:455] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:26.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.278851 17634 sys_catalog.cc:458] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: This master's current role is: FOLLOWER
09:15:26.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.279572 17629 sys_catalog.cc:565] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: configured and running, proceeding with master startup.
09:15:26.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.310132 17645 catalog_manager.cc:1269] Loaded cluster ID: 91258d94f6904e95815ac3730f414487
09:15:26.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.310393 17645 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: loading cluster ID for follower catalog manager: success
09:15:26.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.316758 17645 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: acquiring CA information for follower catalog manager: success
09:15:26.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.320672 17645 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:15:26.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.331141 17544 master_runner.cc:186] Error getting master registration for 127.16.60.125:32949: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111),
09:15:26.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.357120 17544 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:15:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.407958 17650 raft_consensus.cc:493] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 2 FOLLOWER]: Starting pre-election (detected failure of leader d9e5dacfdeb94d4d8bf1f17195e777b4)
09:15:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.408493 17650 raft_consensus.cc:515] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:26.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.410687 17650 leader_election.cc:290] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949), c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777)
09:15:26.417 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.125:32949(127.16.60.125:32949)] connecting to peer
09:15:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:26.417481 16654 leader_election.cc:336] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949): Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111)
09:15:26.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.125:32949(127.16.60.125:32949)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.418 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.16.60.125:32949(127.16.60.125:32949) for tablet Kudu Master: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.423434 17592 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661dc84b98204b03aaa17fef82fe8abc" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "c796ccb9931245c8a473b1b23e405086" is_pre_election: true
09:15:26.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.424403 17592 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 661dc84b98204b03aaa17fef82fe8abc in term 1.
09:15:26.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.425721 16657 leader_election.cc:304] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 661dc84b98204b03aaa17fef82fe8abc, c796ccb9931245c8a473b1b23e405086; no voters: d9e5dacfdeb94d4d8bf1f17195e777b4
09:15:26.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.426566 17650 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 2 FOLLOWER]: Leader pre-election won for term 3
09:15:26.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.426952 17650 raft_consensus.cc:493] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 2 FOLLOWER]: Starting leader election (detected failure of leader d9e5dacfdeb94d4d8bf1f17195e777b4)
09:15:26.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.427309 17650 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 2 FOLLOWER]: Advancing to term 3
09:15:26.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.433204 17650 raft_consensus.cc:515] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:26.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.435261 17650 leader_election.cc:290] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [CANDIDATE]: Term 3 election: Requested vote from peers d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949), c796ccb9931245c8a473b1b23e405086 (127.16.60.124:44777)
09:15:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.435957 17592 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661dc84b98204b03aaa17fef82fe8abc" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "c796ccb9931245c8a473b1b23e405086"
09:15:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.436486 17592 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 1 FOLLOWER]: Advancing to term 3
09:15:26.437 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.125:32949(127.16.60.125:32949)] connecting to peer
09:15:26.438 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.124:44777(127.16.60.124:44777)] connecting to peer
09:15:26.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.125:32949(127.16.60.125:32949)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.439 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.60.124:44777(127.16.60.124:44777)] Successfully connected to peer
09:15:26.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.125:32949: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:26.442911 16654 leader_election.cc:336] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949): Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111)
09:15:26.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.446125 17592 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 661dc84b98204b03aaa17fef82fe8abc in term 3.
09:15:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.447093 16657 leader_election.cc:304] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 661dc84b98204b03aaa17fef82fe8abc, c796ccb9931245c8a473b1b23e405086; no voters: d9e5dacfdeb94d4d8bf1f17195e777b4
09:15:26.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.448264 17650 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 3 FOLLOWER]: Leader election won for term 3
09:15:26.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.449996 17650 raft_consensus.cc:697] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [term 3 LEADER]: Becoming Leader. State: Replica: 661dc84b98204b03aaa17fef82fe8abc, State: Running, Role: LEADER
09:15:26.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.450997 17650 consensus_queue.cc:237] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:26.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.456276 17654 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 661dc84b98204b03aaa17fef82fe8abc. Latest consensus state: current_term: 3 leader_uuid: "661dc84b98204b03aaa17fef82fe8abc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.457201 17654 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: This master's current role is: LEADER
09:15:26.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.462623 17656 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:15:26.464 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0d6ec81b, L:/127.0.0.1:35786 - R:/127.16.60.124:44777] using TOKEN/null
09:15:26.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:26.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.470577 17656 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a]
09:15:26.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.472082 17656 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=f4146e3da09f408794f6caec5133a28a]
09:15:26.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.477089 17656 tablet_loader.cc:96] loaded metadata for tablet 2d3db7ed5f984b45b8fe995cef9d6d8e (table txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a])
09:15:26.477 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.125:32949(127.16.60.125:32949)] connecting to peer
09:15:26.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.125:32949(127.16.60.125:32949)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.125:32949: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.478416 17656 tablet_loader.cc:96] loaded metadata for tablet 49164ca60c4f4a87b9cdf907d2afea7a (table txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a])
09:15:26.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.479722 17656 tablet_loader.cc:96] loaded metadata for tablet 6f5013700e894c0981651f54fb02c2b9 (table kudu_system.kudu_transactions [id=f4146e3da09f408794f6caec5133a28a])
09:15:26.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.480981 17656 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:15:26.482 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:26.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.484879 17656 catalog_manager.cc:1269] Loaded cluster ID: 91258d94f6904e95815ac3730f414487
09:15:26.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.485129 17656 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:15:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.488652 17656 catalog_manager.cc:1514] Loading token signing keys...
09:15:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.491719 17656 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc: Loaded TSK: 0
09:15:26.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.492862 17656 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:15:26.497 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.125:32949(127.16.60.125:32949)] connecting to peer
09:15:26.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.125:32949(127.16.60.125:32949)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.500 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.125:32949: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949
09:15:26.502 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:15:26.502 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=4, TimeoutTracker(timeout=250, elapsed=84), Traces: [0ms] sending RPC to server master-127.16.60.125:32949, [1ms] received response from server master-127.16.60.125:32949: Network error: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [22ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Network error: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949, [25ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: OK, [50ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [50ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Network error: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [65ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: OK, [65ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Network error: Failed to connect to peer master-127.16.60.125:32949(127.16.60.125:32949): Connection refused: /127.16.60.125:32949, [83ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: OK, deferred=Deferred@884101265(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/280876061@1cd2c635 -> (continuation of Deferred@17797969 after retry RPC after error@130159190) -> (continuation of Deferred@559082727 after retry RPC after error@725312918), errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/1864875192@56d3febb -> (continuation of Deferred@17797969 after retry RPC after error@130159190) -> (continuation of Deferred@559082727 after retry RPC after error@725312918))) after lookup
09:15:26.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.542064 17592 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 3 FOLLOWER]: Refusing update from remote peer 661dc84b98204b03aaa17fef82fe8abc: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
09:15:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.546936 17654 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [LEADER]: Connected to new peer: Peer: permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
09:15:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:26.558457 16654 consensus_peers.cc:597] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc -> Peer d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949): Couldn't send request to peer d9e5dacfdeb94d4d8bf1f17195e777b4. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.549640 16669 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55666:
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:15:26.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:26.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:26.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:26.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:15:26.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:26.593948 16669 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:15:26.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.602044 17634 sys_catalog.cc:455] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 661dc84b98204b03aaa17fef82fe8abc. Latest consensus state: current_term: 3 leader_uuid: "661dc84b98204b03aaa17fef82fe8abc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:26.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.602806 17634 sys_catalog.cc:458] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: This master's current role is: FOLLOWER
09:15:26.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.607789 17667 sys_catalog.cc:455] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "661dc84b98204b03aaa17fef82fe8abc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:26.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.608546 17667 sys_catalog.cc:458] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: This master's current role is: FOLLOWER
09:15:26.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.612283 17667 sys_catalog.cc:455] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "661dc84b98204b03aaa17fef82fe8abc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:26.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.612893 17667 sys_catalog.cc:458] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: This master's current role is: FOLLOWER
09:15:26.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.625789 17654 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "661dc84b98204b03aaa17fef82fe8abc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:26.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:26.627813 17654 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc [sys.catalog]: This master's current role is: LEADER
09:15:26.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:26.647974 16700 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:34908 (request call id 5) took 144 ms (client timeout 165 ms). Trace:
09:15:26.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:26.648366 16700 rpcz_store.cc:269] 0130 09:15:26.503648 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:15:26.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:26.503834 (+ 186us) service_pool.cc:225] Handling call
09:15:26.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:26.647743 (+143909us) inbound_call.cc:173] Queueing success response
09:15:26.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":1129,"init.queue_time_us":1119,"init.run_cpu_time_us":19389,"init.run_wall_time_us":97651,"thread_start_us":8241,"threads_started":5}
09:15:27.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:27.056094 16654 consensus_peers.cc:597] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc -> Peer d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949): Couldn't send request to peer d9e5dacfdeb94d4d8bf1f17195e777b4. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:15:27.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:27.129069 17253 heartbeater.cc:344] Connected to a master server at 127.16.60.124:44777
09:15:27.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:27.137682 17581 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" instance_seqno: 1769764518137083) as {username='slave'} at 127.16.60.66:33505; Asking this server to re-register.
09:15:27.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:27.142885 17253 heartbeater.cc:461] Registering TS with master...
09:15:27.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:27.154063 17581 ts_manager.cc:194] Registered new tserver with Master: ac5023c640ae4ccdb6e27ef5cfb9c524 (127.16.60.66:34795)
09:15:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:27.163074 17104 heartbeater.cc:344] Connected to a master server at 127.16.60.124:44777
09:15:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:27.166854 17581 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" instance_seqno: 1769764515901125) as {username='slave'} at 127.16.60.65:33777; Asking this server to re-register.
09:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:27.168354 17104 heartbeater.cc:461] Registering TS with master...
09:15:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:27.171794 17581 ts_manager.cc:194] Registered new tserver with Master: 1aefced973a34ed69a4c40fe377b10aa (127.16.60.65:35089)
09:15:27.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:27.222420 17581 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" instance_seqno: 1769764520537772) as {username='slave'} at 127.16.60.67:38585; Asking this server to re-register.
09:15:27.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:27.224232 17401 heartbeater.cc:461] Registering TS with master...
09:15:27.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:27.227954 17580 ts_manager.cc:194] Registered new tserver with Master: a1eb0155aadb4d13af735b85265f3aed (127.16.60.67:46487)
09:15:27.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:27.428014 16654 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111) [suppressed 11 similar messages]
09:15:27.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:27.539623 16654 consensus_peers.cc:597] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc -> Peer d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949): Couldn't send request to peer d9e5dacfdeb94d4d8bf1f17195e777b4. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:15:28.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.016554 16654 consensus_peers.cc:597] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc -> Peer d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949): Couldn't send request to peer d9e5dacfdeb94d4d8bf1f17195e777b4. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:15:28.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:28.118952 17255 heartbeater.cc:499] Master 127.16.60.126:39927 was elected leader, sending a full tablet report...
09:15:28.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:28.127452 17107 heartbeater.cc:499] Master 127.16.60.126:39927 was elected leader, sending a full tablet report...
09:15:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:28.141450 17403 heartbeater.cc:499] Master 127.16.60.126:39927 was elected leader, sending a full tablet report...
09:15:28.290 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.60.125:32949
09:15:28.290 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:28.291894 16625 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/logs
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data/info.pb
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:15:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.125:32949
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.125
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42103
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:15:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:15:28.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.483052 16654 consensus_peers.cc:597] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc -> Peer d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949): Couldn't send request to peer d9e5dacfdeb94d4d8bf1f17195e777b4. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
09:15:28.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.613075 17700 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:15:28.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.613762 17700 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:15:28.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.614360 17700 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:15:28.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.647218 17700 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:15:28.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.647610 17700 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:15:28.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.647898 17700 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:15:28.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.648188 17700 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:15:28.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.648473 17700 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:15:28.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.648761 17700 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:15:28.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.649008 17700 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43617/jwks.json
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:28.687702 17700 master_runner.cc:386] Master server non-default flags:
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.60.84:33875
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.60.126:39927,127.16.60.125:32949,127.16.60.124:44777
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.60.125:32949
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43617/jwks.json
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data/info.pb
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.60.125
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42103
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:15:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/logs
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Jan 2026 08:43:23 UTC on 5fd53c4cbb9d
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10261
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:15:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:28.689460 17700 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:15:28.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:28.691360 17700 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:15:28.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.702395 17707 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:28.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.993260 16654 consensus_peers.cc:597] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc -> Peer d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949): Couldn't send request to peer d9e5dacfdeb94d4d8bf1f17195e777b4. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
09:15:29.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:29.464555 16654 consensus_peers.cc:597] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc -> Peer d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949): Couldn't send request to peer d9e5dacfdeb94d4d8bf1f17195e777b4. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
09:15:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:29.960732 16654 consensus_peers.cc:597] T 00000000000000000000000000000000 P 661dc84b98204b03aaa17fef82fe8abc -> Peer d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949): Couldn't send request to peer d9e5dacfdeb94d4d8bf1f17195e777b4. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.125:32949: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
09:15:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.707813 17710 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:28.704002 17708 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:15:30.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.081941 17709 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1372 milliseconds
09:15:30.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.082065 17700 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:15:30.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.083556 17700 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:15:30.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.086546 17700 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:15:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.087962 17700 hybrid_clock.cc:648] HybridClock initialized: now 1769764530087898 us; error 69 us; skew 500 ppm
09:15:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.088757 17700 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:15:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.095817 17700 webserver.cc:492] Webserver started at http://127.16.60.125:42103/ using document root <none> and password file <none>
09:15:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.096767 17700 fs_manager.cc:362] Metadata directory not provided
09:15:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.096974 17700 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:15:30.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.104916 17700 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
09:15:30.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.109959 17721 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:15:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.110980 17700 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.111263 17700 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data,/tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4"
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-30 09:15:09 on dist-test-slave-0ffw"
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.113233 17700 fs_report.cc:389] FS layout report
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/wal
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data/data
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:15:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:15:30.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.187219 17700 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:15:30.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.189026 17700 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:15:30.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.189534 17700 kserver.cc:163] Server-wide thread pool size limit: 3276
09:15:30.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.284199 17700 rpc_server.cc:307] RPC server started. Bound to: 127.16.60.125:32949
09:15:30.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.284276 17785 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.60.125:32949 every 8 connection(s)
09:15:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.287349 17700 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskapZmwC/test-tmp/mini-kudu-cluster8276381224180421585/master-1/data/info.pb
09:15:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.295275 16625 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu as pid 17700
09:15:30.296 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:15:30.296 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.60.126:39927
09:15:30.296 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
09:15:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.296658 16625 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu with pid 16637
09:15:30.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.301494 17786 sys_catalog.cc:263] Verifying existing consensus state
09:15:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.310441 17786 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: Bootstrap starting.
09:15:30.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:15:30.346 [ERROR - kudu-nio-1] (Connection.java:452) [peer master-127.16.60.126:39927(127.16.60.126:39927)] unexpected exception from downstream on [id: 0x441b591c, L:/127.0.0.1:34908 - R:/127.16.60.126:39927]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) ~[?:1.8.0_275]
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:192)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379)
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255)
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132)
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
09:15:30.347 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state READY due to: [peer master-127.16.60.126:39927(127.16.60.126:39927)] unexpected exception from downstream on [id: 0x441b591c, L:/127.0.0.1:34908 - R:/127.16.60.126:39927]
09:15:30.347 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.16.60.126:39927(127.16.60.126:39927) for tablet Kudu Master: [peer master-127.16.60.126:39927(127.16.60.126:39927)] unexpected exception from downstream on [id: 0x441b591c, L:/127.0.0.1:34908 - R:/127.16.60.126:39927]
09:15:30.349 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.16.60.126:39927(127.16.60.126:39927)] handling channelInactive
09:15:30.357 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.125:32949(127.16.60.125:32949)] connecting to peer
09:15:30.357 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.359 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.16.60.125:32949(127.16.60.125:32949)] Successfully connected to peer
09:15:30.367 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.16.60.125
09:15:30.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.367713 17786 log.cc:826] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: Log is configured to *not* fsync() on all Append() calls
09:15:30.380 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x8fed063a, L:/127.0.0.1:54444 - R:/127.16.60.125:32949] using SASL/PLAIN
09:15:30.387 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.125:32949: Catalog manager is not initialized. State: Starting
09:15:30.387 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.395020 17397 debug-util.cc:398] Leaking SignalData structure 0x7b08000c8980 after lost signal to thread 17259
09:15:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.396339 17397 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9b00 after lost signal to thread 17400
09:15:30.397 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.402 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.402 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.403 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.403 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.125:32949: Catalog manager is not initialized. State: Starting
09:15:30.403 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.409621 17786 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:15:30.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.410390 17786 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: Bootstrap complete.
09:15:30.417 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.419 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.419 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.420 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.125:32949: Catalog manager is not initialized. State: Starting
09:15:30.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.431265 17786 raft_consensus.cc:359] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.433465 17786 raft_consensus.cc:740] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9e5dacfdeb94d4d8bf1f17195e777b4, State: Initialized, Role: FOLLOWER
09:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.434279 17786 consensus_queue.cc:260] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:30.437 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.436846 17790 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:30.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.439116 17790 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: This master's current role is: FOLLOWER
09:15:30.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.438635 17786 sys_catalog.cc:565] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: configured and running, proceeding with master startup.
09:15:30.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.125:32949: Catalog manager is not initialized. State: Starting
09:15:30.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.440 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.440 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.440 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.457 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.461 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.125:32949: Catalog manager is not initialized. State: Starting
09:15:30.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.480988 17801 catalog_manager.cc:1269] Loaded cluster ID: 91258d94f6904e95815ac3730f414487
09:15:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.481274 17801 catalog_manager.cc:1562] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: loading cluster ID for follower catalog manager: success
09:15:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.488973 17801 catalog_manager.cc:1584] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: acquiring CA information for follower catalog manager: success
09:15:30.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.494803 17801 catalog_manager.cc:1612] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:15:30.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.469949 17700 master_runner.cc:186] Error getting master registration for 127.16.60.126:39927: Network error: Client connection negotiation failed: client connection to 127.16.60.126:39927: connect: Connection refused (error 111),
09:15:30.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.499574 17700 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:15:30.518 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.521 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.521 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.521 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.521 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.522 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=224), Traces: [0ms] sending RPC to server master-127.16.60.126:39927, [50ms] received response from server master-127.16.60.126:39927: Network error: [peer master-127.16.60.126:39927(127.16.60.126:39927)] unexpected exception from downstream on [id: 0x441b591c, L:/127.0.0.1:34908 - R:/127.16.60.126:39927], [50ms] delaying RPC due to: Network error: [peer master-127.16.60.126:39927(127.16.60.126:39927)] unexpected exception from downstream on [id: 0x441b591c, L:/127.0.0.1:34908 - R:/127.16.60.126:39927], [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [90ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [90ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [104ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [106ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [106ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [107ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [164ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [165ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [224ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [224ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [224ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, deferred=Deferred@683908397(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/280876061@1b7b1b5d -> (continuation of Deferred@1400421650 after retry RPC after error@646610222) -> (continuation of Deferred@311735847 after retry RPC after error@1366750939) -> (continuation of Deferred@1206931914 after retry RPC after error@845555479) -> (continuation of Deferred@91523826 after retry RPC after error@1674732713) -> (continuation of Deferred@652810008 after retry RPC after error@632607309), errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/1864875192@436ef13 -> (continuation of Deferred@1400421650 after retry RPC after error@646610222) -> (continuation of Deferred@311735847 after retry RPC after error@1366750939) -> (continuation of Deferred@1206931914 after retry RPC after error@845555479) -> (continuation of Deferred@91523826 after retry RPC after error@1674732713) -> (continuation of Deferred@652810008 after retry RPC after error@632607309)))
09:15:30.523 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=224), Traces: [0ms] sending RPC to server master-127.16.60.126:39927, [50ms] received response from server master-127.16.60.126:39927: Network error: [peer master-127.16.60.126:39927(127.16.60.126:39927)] unexpected exception from downstream on [id: 0x441b591c, L:/127.0.0.1:34908 - R:/127.16.60.126:39927], [50ms] delaying RPC due to: Network error: [peer master-127.16.60.126:39927(127.16.60.126:39927)] unexpected exception from downstream on [id: 0x441b591c, L:/127.0.0.1:34908 - R:/127.16.60.126:39927], [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [61ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [90ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [90ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [104ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [106ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [106ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [107ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [142ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [143ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [163ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [164ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [165ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [223ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [224ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [224ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [224ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, deferred=Deferred@683908397(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/280876061@1b7b1b5d -> (continuation of Deferred@1400421650 after retry RPC after error@646610222) -> (continuation of Deferred@311735847 after retry RPC after error@1366750939) -> (continuation of Deferred@1206931914 after retry RPC after error@845555479) -> (continuation of Deferred@91523826 after retry RPC after error@1674732713) -> (continuation of Deferred@652810008 after retry RPC after error@632607309), errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/1864875192@436ef13 -> (continuation of Deferred@1400421650 after retry RPC after error@646610222) -> (continuation of Deferred@311735847 after retry RPC after error@1366750939) -> (continuation of Deferred@1206931914 after retry RPC after error@845555479) -> (continuation of Deferred@91523826 after retry RPC after error@1674732713) -> (continuation of Deferred@652810008 after retry RPC after error@632607309)))
09:15:30.537 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.539 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.544 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.557 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.560 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.560 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.560 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.564 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.577 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.583017 17804 raft_consensus.cc:493] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 661dc84b98204b03aaa17fef82fe8abc)
09:15:30.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.583631 17804 raft_consensus.cc:515] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:30.597 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.602 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.603240 17567 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.60.126:39927: connect: Connection refused (error 111)
09:15:30.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.603374 17804 leader_election.cc:290] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927), d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949)
09:15:30.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.611191 17751 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c796ccb9931245c8a473b1b23e405086" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" is_pre_election: true
09:15:30.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.612129 17751 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c796ccb9931245c8a473b1b23e405086 in term 2.
09:15:30.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.613428 17565 leader_election.cc:304] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c796ccb9931245c8a473b1b23e405086, d9e5dacfdeb94d4d8bf1f17195e777b4; no voters:
09:15:30.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.614285 17804 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 3 FOLLOWER]: Leader pre-election won for term 4
09:15:30.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.614500 17567 leader_election.cc:336] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927): Network error: Client connection negotiation failed: client connection to 127.16.60.126:39927: connect: Connection refused (error 111)
09:15:30.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.614620 17804 raft_consensus.cc:493] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 661dc84b98204b03aaa17fef82fe8abc)
09:15:30.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.615000 17804 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 3 FOLLOWER]: Advancing to term 4
09:15:30.617 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.625644 17804 raft_consensus.cc:515] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.627720 17804 leader_election.cc:290] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [CANDIDATE]: Term 4 election: Requested vote from peers 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927), d9e5dacfdeb94d4d8bf1f17195e777b4 (127.16.60.125:32949)
09:15:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.630533 17751 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c796ccb9931245c8a473b1b23e405086" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4"
09:15:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.631078 17751 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 2 FOLLOWER]: Advancing to term 4
09:15:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.632767 17567 leader_election.cc:336] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927): Network error: Client connection negotiation failed: client connection to 127.16.60.126:39927: connect: Connection refused (error 111)
09:15:30.634 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.640889 17751 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c796ccb9931245c8a473b1b23e405086 in term 4.
09:15:30.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.641988 17565 leader_election.cc:304] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c796ccb9931245c8a473b1b23e405086, d9e5dacfdeb94d4d8bf1f17195e777b4; no voters: 661dc84b98204b03aaa17fef82fe8abc
09:15:30.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.642802 17804 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 4 FOLLOWER]: Leader election won for term 4
09:15:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.644695 17804 raft_consensus.cc:697] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [term 4 LEADER]: Becoming Leader. State: Replica: c796ccb9931245c8a473b1b23e405086, State: Running, Role: LEADER
09:15:30.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.645665 17804 consensus_queue.cc:237] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 3.13, Last appended by leader: 13, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } }
09:15:30.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.653177 17808 sys_catalog.cc:455] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c796ccb9931245c8a473b1b23e405086. Latest consensus state: current_term: 4 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:30.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.654381 17808 sys_catalog.cc:458] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: This master's current role is: LEADER
09:15:30.657 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.662 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.663400 17810 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:15:30.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.671054 17810 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a]
09:15:30.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.672713 17810 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=f4146e3da09f408794f6caec5133a28a]
09:15:30.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.679286 17810 tablet_loader.cc:96] loaded metadata for tablet 2d3db7ed5f984b45b8fe995cef9d6d8e (table txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a])
09:15:30.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.680596 17810 tablet_loader.cc:96] loaded metadata for tablet 49164ca60c4f4a87b9cdf907d2afea7a (table txn_manager_fallback_rolling [id=8443fe4bd22a423ba4cb25bc662d8a4a])
09:15:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.681855 17810 tablet_loader.cc:96] loaded metadata for tablet 6f5013700e894c0981651f54fb02c2b9 (table kudu_system.kudu_transactions [id=f4146e3da09f408794f6caec5133a28a])
09:15:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.683108 17810 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:15:30.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.687196 17810 catalog_manager.cc:1269] Loaded cluster ID: 91258d94f6904e95815ac3730f414487
09:15:30.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.687479 17810 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:15:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.692127 17810 catalog_manager.cc:1514] Loading token signing keys...
09:15:30.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.696302 17810 catalog_manager.cc:6038] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086: Loaded TSK: 0
09:15:30.697 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.697536 17810 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:15:30.704 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777; will retry
09:15:30.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.744796 17567 consensus_peers.cc:597] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 -> Peer 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927): Couldn't send request to peer 661dc84b98204b03aaa17fef82fe8abc. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.126:39927: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:15:30.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.759938 17751 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [term 4 FOLLOWER]: Refusing update from remote peer c796ccb9931245c8a473b1b23e405086: Log matching property violated. Preceding OpId in replica: term: 2 index: 12. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
09:15:30.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.761620 17808 consensus_queue.cc:1048] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 12, Time since last communication: 0.000s
09:15:30.777 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.16.60.126:39927(127.16.60.126:39927)] connecting to peer
09:15:30.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.126:39927(127.16.60.126:39927)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.126:39927: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927
09:15:30.780 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:15:30.781 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [7ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [7ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [27ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [27ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [65ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [97ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [97ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [164ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [167ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [167ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, deferred=Deferred@142039217(state=PENDING, result=null, callback=(continuation of Deferred@829935393 after retry RPC after error@1601365691) -> (continuation of Deferred@1751365678 after retry RPC after error@1524877123) -> (continuation of Deferred@1440880121 after retry RPC after error@502101696) -> (continuation of Deferred@1757024155 after retry RPC after error@749143056) -> (continuation of Deferred@1764721056 after retry RPC after error@528654643) -> (continuation of Deferred@670868367 after retry RPC after error@871343332) -> (continuation of Deferred@651694520 after retry RPC after error@752465769), errback=(continuation of Deferred@829935393 after retry RPC after error@1601365691) -> (continuation of Deferred@1751365678 after retry RPC after error@1524877123) -> (continuation of Deferred@1440880121 after retry RPC after error@502101696) -> (continuation of Deferred@1757024155 after retry RPC after error@749143056) -> (continuation of Deferred@1764721056 after retry RPC after error@528654643) -> (continuation of Deferred@670868367 after retry RPC after error@871343332) -> (continuation of Deferred@651694520 after retry RPC after error@752465769))) after lookup
09:15:30.797 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [2ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [3ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [7ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [7ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [23ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [24ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [27ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [27ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [42ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [43ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [44ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [62ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [65ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [81ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [84ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [97ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [97ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [122ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [162ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [164ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [167ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.125:32949: OK, [167ms] delaying RPC due to: Service unavailable: Master config (127.16.60.125:32949,127.16.60.126:39927,127.16.60.124:44777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.125:32949, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.126:39927, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.60.124:44777, [241ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.126:39927: Network error: Failed to connect to peer master-127.16.60.126:39927(127.16.60.126:39927): Connection refused: /127.16.60.126:39927, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.60.124:44777: OK, [245ms] sending RPC to server master-127.16.60.124:44777, deferred=Deferred@142039217(state=PENDING, result=null, callback=(continuation of Deferred@829935393 after retry RPC after error@1601365691) -> (continuation of Deferred@1751365678 after retry RPC after error@1524877123) -> (continuation of Deferred@1440880121 after retry RPC after error@502101696) -> (continuation of Deferred@1757024155 after retry RPC after error@749143056) -> (continuation of Deferred@1764721056 after retry RPC after error@528654643) -> (continuation of Deferred@670868367 after retry RPC after error@871343332) -> (continuation of Deferred@651694520 after retry RPC after error@752465769) -> (continuation of Deferred@1727560476 after retry RPC@1075991402), errback=(continuation of Deferred@829935393 after retry RPC after error@1601365691) -> (continuation of Deferred@1751365678 after retry RPC after error@1524877123) -> (continuation of Deferred@1440880121 after retry RPC after error@502101696) -> (continuation of Deferred@1757024155 after retry RPC after error@749143056) -> (continuation of Deferred@1764721056 after retry RPC after error@528654643) -> (continuation of Deferred@670868367 after retry RPC after error@871343332) -> (continuation of Deferred@651694520 after retry RPC after error@752465769) -> (continuation of Deferred@1727560476 after retry RPC@1075991402)))
09:15:30.798 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.60.125:32949: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=1, TimeoutTracker(timeout=11, elapsed=21), Traces: [0ms] sending RPC to server master-127.16.60.125:32949, deferred=Deferred@244045267(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.16.60.125:32949, errback=org.apache.kudu.client.ConnectToCluster$1@3e617bd2 -> ConnectToMasterErrCB for 127.16.60.125:32949))
09:15:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.804271 17612 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:35786 (request call id 17) took 17 ms (client timeout 5 ms). Trace:
09:15:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.804746 17612 rpcz_store.cc:269] 0130 09:15:30.786611 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:15:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:30.786766 (+ 155us) service_pool.cc:225] Handling call
09:15:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:30.804022 (+ 17256us) inbound_call.cc:173] Queueing success response
09:15:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":14952,"thread_start_us":16568,"threads_started":1}
09:15:30.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.835317 17814 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:30.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.836261 17814 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: This master's current role is: FOLLOWER
09:15:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.837231 17790 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c796ccb9931245c8a473b1b23e405086. Latest consensus state: current_term: 4 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.838158 17790 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: This master's current role is: FOLLOWER
09:15:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.839979 17741 rpcz_store.cc:267] Call kudu.master.MasterService.ConnectToMaster from 127.0.0.1:54444 (request call id 13) took 59 ms (client timeout 11 ms). Trace:
09:15:30.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.840356 17741 rpcz_store.cc:269] 0130 09:15:30.780490 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:15:30.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:30.782960 (+ 2470us) service_pool.cc:225] Handling call
09:15:30.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:30.822279 (+ 39319us) spinlock_profiling.cc:243] Waited 27.8 ms on lock 0x7b54000104d4. stack: 00007fbb61c0a373 00007fbb61c0a270 00007fbb6177f574 0000000000508744 00000000005086d9 00007fbb68c5c0c0 00007fbb68cfabba 00007fbb6a9f9eac 00007fbb6aaf73c9 00007fbb68713a24 00007fbb687139b2 00007fbb687138e1 00007fbb6871385c 00007fbb68712b12 00007fbb669f034c 00007fbb669ef786
09:15:30.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:30.839474 (+ 17195us) spinlock_profiling.cc:243] Waited 11.3 ms on lock 0x7b54000104d4. stack: 00007fbb61c0a373 00007fbb61c0a270 00007fbb6177f574 0000000000508744 00000000005086d9 00007fbb68c5b5a1 00007fbb68cfac79 00007fbb6aab7e56 00007fbb6aaf74f7 00007fbb68713a24 00007fbb687139b2 00007fbb687138e1 00007fbb6871385c 00007fbb68712b12 00007fbb669f034c 00007fbb669ef786
09:15:30.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:30.839761 (+ 287us) inbound_call.cc:173] Queueing success response
09:15:30.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":89998592}
09:15:30.842 [DEBUG - kudu-nio-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ConnectToMaster, tablet=null, attempt=0, TimeoutTracker(timeout=0, elapsed=44), Traces: [0ms] received response from server master-127.16.60.125:32949: OK, deferred=null) with no deferred attached!
09:15:30.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.842415 17816 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:30.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.843875 17816 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9e5dacfdeb94d4d8bf1f17195e777b4 [sys.catalog]: This master's current role is: FOLLOWER
09:15:30.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.844440 17804 sys_catalog.cc:455] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "c796ccb9931245c8a473b1b23e405086" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "661dc84b98204b03aaa17fef82fe8abc" member_type: VOTER last_known_addr { host: "127.16.60.126" port: 39927 } } peers { permanent_uuid: "d9e5dacfdeb94d4d8bf1f17195e777b4" member_type: VOTER last_known_addr { host: "127.16.60.125" port: 32949 } } peers { permanent_uuid: "c796ccb9931245c8a473b1b23e405086" member_type: VOTER last_known_addr { host: "127.16.60.124" port: 44777 } } }
09:15:30.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.845145 17804 sys_catalog.cc:458] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 [sys.catalog]: This master's current role is: LEADER
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:30.981801 17580 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35796:
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:15:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:15:30.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:30.989281 17580 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:15:31.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:31.055156 17612 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:35786 (request call id 18) took 235 ms (client timeout 250 ms). Trace:
09:15:31.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:31.055594 17612 rpcz_store.cc:269] 0130 09:15:30.819396 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:15:31.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:30.821291 (+ 1895us) service_pool.cc:225] Handling call
09:15:31.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0130 09:15:31.055081 (+233790us) inbound_call.cc:173] Queueing success response
09:15:31.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:15:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.116573 17136 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.233s from last keepalive heartbeat (effective timeout is 1.000s)
09:15:31.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:31.216512 17148 outbound_call.cc:321] RPC callback for RPC call kudu.master.MasterService.GetTabletLocations -> {remote=127.16.60.124:44777, user_credentials={real_user=slave}} blocked reactor thread for 61269.1us
09:15:31.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.246253 17254 heartbeater.cc:344] Connected to a master server at 127.16.60.125:32949
09:15:31.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:31.259621 17830 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
09:15:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.271268 17741 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac5023c640ae4ccdb6e27ef5cfb9c524" instance_seqno: 1769764518137083) as {username='slave'} at 127.16.60.66:54621; Asking this server to re-register.
09:15:31.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:31.287936 17567 consensus_peers.cc:597] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 -> Peer 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927): Couldn't send request to peer 661dc84b98204b03aaa17fef82fe8abc. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.126:39927: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:15:31.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.300670 17254 heartbeater.cc:461] Registering TS with master...
09:15:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.317044 17105 heartbeater.cc:344] Connected to a master server at 127.16.60.125:32949
09:15:31.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.323191 17741 ts_manager.cc:194] Registered new tserver with Master: ac5023c640ae4ccdb6e27ef5cfb9c524 (127.16.60.66:34795)
09:15:31.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.331905 17741 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1aefced973a34ed69a4c40fe377b10aa" instance_seqno: 1769764515901125) as {username='slave'} at 127.16.60.65:35373; Asking this server to re-register.
09:15:31.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.336704 17402 heartbeater.cc:344] Connected to a master server at 127.16.60.125:32949
09:15:31.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.360775 17544 master_runner.cc:186] Error getting master registration for 127.16.60.126:39927: Network error: Client connection negotiation failed: client connection to 127.16.60.126:39927: connect: Connection refused (error 111),
09:15:31.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.361207 17544 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:15:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.378896 17105 heartbeater.cc:461] Registering TS with master...
09:15:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.383312 17740 ts_manager.cc:194] Registered new tserver with Master: 1aefced973a34ed69a4c40fe377b10aa (127.16.60.65:35089)
09:15:31.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.393859 17741 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a1eb0155aadb4d13af735b85265f3aed" instance_seqno: 1769764520537772) as {username='slave'} at 127.16.60.67:57171; Asking this server to re-register.
09:15:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.395453 17402 heartbeater.cc:461] Registering TS with master...
09:15:31.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:31.399888 17740 ts_manager.cc:194] Registered new tserver with Master: a1eb0155aadb4d13af735b85265f3aed (127.16.60.67:46487)
09:15:31.585 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: ac5023c640ae4ccdb6e27ef5cfb9c524 (127.16.60.66:34795): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1769764522 last_transition_timestamp: 1769764531
09:15:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:31.827966 17567 consensus_peers.cc:597] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 -> Peer 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927): Couldn't send request to peer 661dc84b98204b03aaa17fef82fe8abc. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.126:39927: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:15:32.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:32.151659 17253 heartbeater.cc:499] Master 127.16.60.124:44777 was elected leader, sending a full tablet report...
09:15:32.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:32.195849 17104 heartbeater.cc:499] Master 127.16.60.124:44777 was elected leader, sending a full tablet report...
09:15:32.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:32.256340 17401 heartbeater.cc:499] Master 127.16.60.124:44777 was elected leader, sending a full tablet report...
09:15:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:32.308892 17567 consensus_peers.cc:597] T 00000000000000000000000000000000 P c796ccb9931245c8a473b1b23e405086 -> Peer 661dc84b98204b03aaa17fef82fe8abc (127.16.60.126:39927): Couldn't send request to peer 661dc84b98204b03aaa17fef82fe8abc. Status: Network error: Client connection negotiation failed: client connection to 127.16.60.126:39927: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:15:32.353 [DEBUG - main] (Connection.java:575) [peer 1aefced973a34ed69a4c40fe377b10aa(127.16.60.65:35089)] disconnecting while in state READY
09:15:32.353 [DEBUG - main] (Connection.java:575) [peer master-127.16.60.125:32949(127.16.60.125:32949)] disconnecting while in state READY
09:15:32.353 [DEBUG - main] (Connection.java:575) [peer master-127.16.60.124:44777(127.16.60.124:44777)] disconnecting while in state READY
09:15:32.354 [DEBUG - main] (Connection.java:575) [peer master-127.16.60.126:39927(127.16.60.126:39927)] disconnecting while in state TERMINATED
09:15:32.355 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 1aefced973a34ed69a4c40fe377b10aa(127.16.60.65:35089)] handling channelInactive
09:15:32.355 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 1aefced973a34ed69a4c40fe377b10aa(127.16.60.65:35089)] cleaning up while in state READY due to: connection closed
09:15:32.355 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.16.60.125:32949(127.16.60.125:32949)] handling channelInactive
09:15:32.356 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:15:32.356 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.60.125:32949(127.16.60.125:32949)] cleaning up while in state READY due to: connection closed
09:15:32.356 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.60.124:44777(127.16.60.124:44777)] handling channelInactive
09:15:32.356 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.60.124:44777(127.16.60.124:44777)] cleaning up while in state READY due to: connection closed
09:15:32.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:32.357568 16625 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu with pid 16938
09:15:32.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:32.412783 16625 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu with pid 17109
09:15:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:32.465706 16625 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu with pid 17258
09:15:32.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:32.499204 16625 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu with pid 17700
09:15:32.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260130 09:15:32.526940 17565 connection.cc:537] client connection to 127.16.60.125:32949 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:15:32.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260130 09:15:32.527477 16625 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskapZmwC/build/tsan/bin/kudu with pid 17544
09:15:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-30T09:15:33Z chronyd exiting
09:15:34.754 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)