21:04:03.038 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:04:03.039 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskfRBwJn/test-tmp
21:04:03.039 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
21:04:03.039 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskfRBwJn/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
21:04:03.040 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:04:03.389 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:04:03.389 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:04:03.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.392280 29680 mini_oidc.cc:150] Starting JWKS server
21:04:03.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.394938 29680 webserver.cc:492] Webserver started at http://127.0.0.1:43225/ using document root <none> and password file <none>
21:04:03.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.395219 29680 mini_oidc.cc:186] Starting OIDC Discovery server
21:04:03.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.395735 29680 webserver.cc:492] Webserver started at http://127.0.0.1:39203/ using document root <none> and password file <none>
21:04:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-10T21:04:03Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:04:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-10T21:04:03Z Disabled control of system clock
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.428321 29680 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/wal
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/data
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/logs
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/data/info.pb
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.62:33507
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.62
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:04:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:04:03.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.762598 29691 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:03.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.762940 29691 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:03.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.763022 29691 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.771756 29691 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:04:03.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.771873 29691 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:04:03.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.771922 29691 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:03.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.771965 29691 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:03.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.772017 29691 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:04:03.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.772058 29691 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:03.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.772107 29691 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43225/jwks.json
21:04:03.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.782485 29691 master_runner.cc:386] Master server non-default flags:
21:04:03.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:03.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:03.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:03.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:03.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:04:03.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/data
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/wal
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.62:33507
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/data/info.pb
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.62
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/logs
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 20:43:25 UTC on 24a791456cd2
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9452
21:04:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:03.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.784195 29691 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.786065 29691 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.795276 29697 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.795257 29696 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.795464 29699 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.796437 29691 server_base.cc:1047] running on GCE node
21:04:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.797268 29691 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.798869 29691 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.800189 29691 hybrid_clock.cc:648] HybridClock initialized: now 1765400643800081 us; error 49 us; skew 500 ppm
21:04:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.800573 29691 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:03.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.803189 29691 webserver.cc:492] Webserver started at http://127.28.252.62:46835/ using document root <none> and password file <none>
21:04:03.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.804072 29691 fs_manager.cc:362] Metadata directory not provided
21:04:03.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.804235 29691 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:03.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.804611 29691 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.807236 29691 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/data/instance:
21:04:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b7ebd5a66a24240b96752e63ff38268"
21:04:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:03 on dist-test-slave-nl76"
21:04:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.808045 29691 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/wal/instance:
21:04:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b7ebd5a66a24240b96752e63ff38268"
21:04:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:03 on dist-test-slave-nl76"
21:04:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.813570 29691 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
21:04:03.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.816998 29705 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.818367 29691 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
21:04:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.818598 29691 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/data,/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/wal
21:04:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b7ebd5a66a24240b96752e63ff38268"
21:04:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:03 on dist-test-slave-nl76"
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.818809 29691 fs_report.cc:389] FS layout report
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/wal
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/wal
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/data/data
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:03.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.843266 29691 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:03.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.844239 29691 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:03.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.844578 29691 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:03.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.869637 29691 rpc_server.cc:307] RPC server started. Bound to: 127.28.252.62:33507
21:04:03.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.869673 29766 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.252.62:33507 every 8 connection(s)
21:04:03.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.871256 29691 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/data/info.pb
21:04:03.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.875494 29767 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:03.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.877224 29680 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu as pid 29691
21:04:03.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.877539 29680 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-0/wal/instance
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.879865 29680 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/wal
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/data
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/logs
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/data/info.pb
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.61:36963
21:04:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.61
21:04:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json
21:04:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:04:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:04:03.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.881491 29767 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } has no permanent_uuid. Determining permanent_uuid...
21:04:03.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.899641 29767 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } has no permanent_uuid. Determining permanent_uuid...
21:04:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.901021 29709 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.252.61:36963: connect: Connection refused (error 111)
21:04:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.902843 29767 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.61:36963: Network error: Client connection negotiation failed: client connection to 127.28.252.61:36963: connect: Connection refused (error 111)
21:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:03.952248 29767 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } attempt: 1
21:04:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:03.954332 29767 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.61:36963: Network error: Client connection negotiation failed: client connection to 127.28.252.61:36963: connect: Connection refused (error 111)
21:04:04.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.022640 29767 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } attempt: 2
21:04:04.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.025064 29767 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.61:36963: Network error: Client connection negotiation failed: client connection to 127.28.252.61:36963: connect: Connection refused (error 111)
21:04:04.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.116343 29767 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } attempt: 3
21:04:04.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.118777 29767 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.61:36963: Network error: Client connection negotiation failed: client connection to 127.28.252.61:36963: connect: Connection refused (error 111)
21:04:04.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.224737 29769 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:04.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.225124 29769 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:04.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.225215 29769 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:04.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.233886 29769 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:04:04.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.234018 29769 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:04:04.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.234069 29769 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:04.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.234287 29769 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:04.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.234386 29769 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:04:04.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.234440 29769 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:04.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.234489 29769 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43225/jwks.json
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.245165 29769 master_runner.cc:386] Master server non-default flags:
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/data
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/wal
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.61:36963
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/data/info.pb
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.61
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/logs
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 20:43:25 UTC on 24a791456cd2
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9452
21:04:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.246927 29769 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:04.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.248857 29769 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.257790 29778 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.257778 29777 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:04.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.258973 29780 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:04.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.259606 29769 server_base.cc:1047] running on GCE node
21:04:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.260474 29769 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:04.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.262102 29769 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:04.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.262104 29767 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } attempt: 4
21:04:04.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.263406 29769 hybrid_clock.cc:648] HybridClock initialized: now 1765400644263324 us; error 58 us; skew 500 ppm
21:04:04.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.263788 29769 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:04.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.264168 29767 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.61:36963: Network error: Client connection negotiation failed: client connection to 127.28.252.61:36963: connect: Connection refused (error 111)
21:04:04.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.266214 29769 webserver.cc:492] Webserver started at http://127.28.252.61:35771/ using document root <none> and password file <none>
21:04:04.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.267036 29769 fs_manager.cc:362] Metadata directory not provided
21:04:04.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.267168 29769 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:04.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.267503 29769 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:04.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.270176 29769 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/data/instance:
21:04:04.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21371941629d4fd4b6222c61bce67340"
21:04:04.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:04 on dist-test-slave-nl76"
21:04:04.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.271078 29769 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/wal/instance:
21:04:04.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21371941629d4fd4b6222c61bce67340"
21:04:04.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:04 on dist-test-slave-nl76"
21:04:04.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.276921 29769 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
21:04:04.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.281050 29786 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:04.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.282758 29769 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.282948 29769 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/data,/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/wal
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21371941629d4fd4b6222c61bce67340"
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:04 on dist-test-slave-nl76"
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.283176 29769 fs_report.cc:389] FS layout report
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/wal
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/wal
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/data/data
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:04.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.299101 29769 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:04.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.300058 29769 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:04.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.300356 29769 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:04.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.325117 29769 rpc_server.cc:307] RPC server started. Bound to: 127.28.252.61:36963
21:04:04.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.325186 29847 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.252.61:36963 every 8 connection(s)
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.326860 29769 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/data/info.pb
21:04:04.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.330901 29680 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu as pid 29769
21:04:04.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.331051 29680 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-1/wal/instance
21:04:04.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.331391 29848 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.332016 29680 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/wal
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/data
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/logs
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/data/info.pb
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.60:39391
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.60
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json
21:04:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:04:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:04:04.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.338997 29848 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } has no permanent_uuid. Determining permanent_uuid...
21:04:04.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.353605 29848 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } has no permanent_uuid. Determining permanent_uuid...
21:04:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.363327 29848 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } has no permanent_uuid. Determining permanent_uuid...
21:04:04.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.364503 29790 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:04.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.366101 29848 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.60:39391: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:04.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.415472 29848 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } attempt: 1
21:04:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.417660 29848 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.60:39391: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.485941 29848 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } attempt: 2
21:04:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.488036 29848 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.60:39391: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:04.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.563460 29767 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } attempt: 5
21:04:04.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.569952 29767 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } has no permanent_uuid. Determining permanent_uuid...
21:04:04.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.572033 29767 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.60:39391: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:04.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.579326 29848 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } attempt: 3
21:04:04.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.581509 29848 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.60:39391: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.623293 29767 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } attempt: 1
21:04:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.625574 29767 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.60:39391: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.687544 29850 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.687889 29850 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.688010 29850 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.693856 29767 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } attempt: 2
21:04:04.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.695956 29767 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.60:39391: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.697165 29850 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:04:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.697265 29850 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:04:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.697315 29850 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.697356 29850 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.697417 29850 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:04:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.697458 29850 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.697501 29850 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43225/jwks.json
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.707935 29850 master_runner.cc:386] Master server non-default flags:
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/data
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/wal
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.60:39391
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/data/info.pb
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.60
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/logs
21:04:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:04:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 20:43:25 UTC on 24a791456cd2
21:04:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9452
21:04:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.709713 29850 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:04.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.711585 29850 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:04.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.720199 29858 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:04.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.720124 29859 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:04.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.720878 29861 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:04.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.721698 29850 server_base.cc:1047] running on GCE node
21:04:04.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.723380 29850 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:04.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.724745 29848 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } attempt: 4
21:04:04.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.725128 29850 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:04.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.726397 29850 hybrid_clock.cc:648] HybridClock initialized: now 1765400644726265 us; error 93 us; skew 500 ppm
21:04:04.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.726749 29848 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.60:39391: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:04.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.726807 29850 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:04.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.729416 29850 webserver.cc:492] Webserver started at http://127.28.252.60:42755/ using document root <none> and password file <none>
21:04:04.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.730244 29850 fs_manager.cc:362] Metadata directory not provided
21:04:04.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.730388 29850 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:04.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.730798 29850 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:04.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.733335 29850 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/data/instance:
21:04:04.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c16a40ef4eea44bf9ee93bded181a256"
21:04:04.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:04 on dist-test-slave-nl76"
21:04:04.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.734158 29850 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/wal/instance:
21:04:04.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c16a40ef4eea44bf9ee93bded181a256"
21:04:04.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:04 on dist-test-slave-nl76"
21:04:04.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.739816 29850 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.004s sys 0.004s
21:04:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.743363 29867 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.744980 29850 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:04:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.745204 29850 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/data,/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/wal
21:04:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c16a40ef4eea44bf9ee93bded181a256"
21:04:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:04 on dist-test-slave-nl76"
21:04:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.745434 29850 fs_report.cc:389] FS layout report
21:04:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/wal
21:04:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/wal
21:04:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/data/data
21:04:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.802232 29767 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } attempt: 3
21:04:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.804054 29767 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.252.60:39391: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:04.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.809815 29850 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:04.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.810786 29850 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:04.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.811091 29850 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:04.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.834619 29850 rpc_server.cc:307] RPC server started. Bound to: 127.28.252.60:39391
21:04:04.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.834686 29928 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.252.60:39391 every 8 connection(s)
21:04:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.836256 29850 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/data/info.pb
21:04:04.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.840879 29929 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:04.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.843838 29680 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu as pid 29850
21:04:04.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.844060 29680 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/master-2/wal/instance
21:04:04.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.848714 29929 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } has no permanent_uuid. Determining permanent_uuid...
21:04:04.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.863859 29929 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } has no permanent_uuid. Determining permanent_uuid...
21:04:04.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.871655 29929 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } has no permanent_uuid. Determining permanent_uuid...
21:04:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.886413 29929 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256: Bootstrap starting.
21:04:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.890125 29929 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256: Neither blocks nor log segments found. Creating new log.
21:04:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.891342 29929 log.cc:826] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256: Log is configured to *not* fsync() on all Append() calls
21:04:04.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.893802 29929 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256: No bootstrap required, opened a new log
21:04:04.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.899143 29929 raft_consensus.cc:359] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:04.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.899479 29929 raft_consensus.cc:385] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:04.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.899597 29929 raft_consensus.cc:740] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c16a40ef4eea44bf9ee93bded181a256, State: Initialized, Role: FOLLOWER
21:04:04.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.900341 29929 consensus_queue.cc:260] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.901784 29938 sys_catalog.cc:455] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.902074 29938 sys_catalog.cc:458] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [sys.catalog]: This master's current role is: FOLLOWER
21:04:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.902964 29929 sys_catalog.cc:565] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [sys.catalog]: configured and running, proceeding with master startup.
21:04:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.913866 29949 catalog_manager.cc:1568] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:04:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:04.914027 29949 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:04:04.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.911028 29850 master_runner.cc:186] Error getting master registration for 127.28.252.62:33507: OK, instance_id { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" instance_seqno: 1765400643863776 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:04:04.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.915613 29850 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.917481 29938 raft_consensus.cc:493] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.917761 29938 raft_consensus.cc:515] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:04.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.919412 29938 leader_election.cc:290] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b7ebd5a66a24240b96752e63ff38268 (127.28.252.62:33507), 21371941629d4fd4b6222c61bce67340 (127.28.252.61:36963)
21:04:04.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.981317 29767 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } attempt: 4
21:04:04.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:04.996587 29767 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268: Bootstrap starting.
21:04:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.000496 29767 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268: Neither blocks nor log segments found. Creating new log.
21:04:05.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.001863 29767 log.cc:826] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268: Log is configured to *not* fsync() on all Append() calls
21:04:05.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.004963 29767 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268: No bootstrap required, opened a new log
21:04:05.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.013588 29767 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:05.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.013986 29767 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:05.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.014130 29767 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b7ebd5a66a24240b96752e63ff38268, State: Initialized, Role: FOLLOWER
21:04:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.015296 29767 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.016819 29952 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.017127 29952 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [sys.catalog]: This master's current role is: FOLLOWER
21:04:05.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.018047 29767 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [sys.catalog]: configured and running, proceeding with master startup.
21:04:05.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.018226 29732 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c16a40ef4eea44bf9ee93bded181a256" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b7ebd5a66a24240b96752e63ff38268" is_pre_election: true
21:04:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.018929 29732 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c16a40ef4eea44bf9ee93bded181a256 in term 0.
21:04:05.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.019850 29869 leader_election.cc:304] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b7ebd5a66a24240b96752e63ff38268, c16a40ef4eea44bf9ee93bded181a256; no voters:
21:04:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.020424 29938 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.020617 29938 raft_consensus.cc:493] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.020723 29938 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [term 0 FOLLOWER]: Advancing to term 1
21:04:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.024067 29938 raft_consensus.cc:515] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.026109 29848 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } attempt: 5
21:04:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.026206 29938 leader_election.cc:290] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [CANDIDATE]: Term 1 election: Requested vote from peers 6b7ebd5a66a24240b96752e63ff38268 (127.28.252.62:33507), 21371941629d4fd4b6222c61bce67340 (127.28.252.61:36963)
21:04:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.028690 29732 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c16a40ef4eea44bf9ee93bded181a256" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b7ebd5a66a24240b96752e63ff38268"
21:04:05.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.029114 29732 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 0 FOLLOWER]: Advancing to term 1
21:04:05.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.032783 29732 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c16a40ef4eea44bf9ee93bded181a256 in term 1.
21:04:05.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.036585 29869 leader_election.cc:304] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b7ebd5a66a24240b96752e63ff38268, c16a40ef4eea44bf9ee93bded181a256; no voters:
21:04:05.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.037151 29938 raft_consensus.cc:2804] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [term 1 FOLLOWER]: Leader election won for term 1
21:04:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.047480 29938 raft_consensus.cc:697] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [term 1 LEADER]: Becoming Leader. State: Replica: c16a40ef4eea44bf9ee93bded181a256, State: Running, Role: LEADER
21:04:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.048383 29938 consensus_queue.cc:237] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:05.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.053359 29848 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340: Bootstrap starting.
21:04:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.055056 29958 sys_catalog.cc:455] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c16a40ef4eea44bf9ee93bded181a256. Latest consensus state: current_term: 1 leader_uuid: "c16a40ef4eea44bf9ee93bded181a256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.055649 29958 sys_catalog.cc:458] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [sys.catalog]: This master's current role is: LEADER
21:04:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.060463 29960 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:04:05.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.063192 29873 tablet.cc:2378] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:05.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.063501 29960 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:04:05.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.064846 29848 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340: Neither blocks nor log segments found. Creating new log.
21:04:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.072849 29848 log.cc:826] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340: Log is configured to *not* fsync() on all Append() calls
21:04:05.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.080153 29732 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 1 FOLLOWER]: Refusing update from remote peer c16a40ef4eea44bf9ee93bded181a256: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:05.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.081573 29958 consensus_queue.cc:1048] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:05.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.096355 29848 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340: No bootstrap required, opened a new log
21:04:05.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.097836 29711 tablet.cc:2378] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:05.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.104054 29952 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c16a40ef4eea44bf9ee93bded181a256. Latest consensus state: current_term: 1 leader_uuid: "c16a40ef4eea44bf9ee93bded181a256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:05.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.104342 29952 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [sys.catalog]: This master's current role is: FOLLOWER
21:04:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.108343 29963 mvcc.cc:204] Tried to move back new op lower bound from 7231081042230767616 to 7231081042124902400. Current Snapshot: MvccSnapshot[applied={T|T < 7231081042230767616}]
21:04:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.119897 29967 mvcc.cc:204] Tried to move back new op lower bound from 7231081042230767616 to 7231081042124902400. Current Snapshot: MvccSnapshot[applied={T|T < 7231081042230767616}]
21:04:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.121629 29938 sys_catalog.cc:455] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c16a40ef4eea44bf9ee93bded181a256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.123116 29938 sys_catalog.cc:458] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [sys.catalog]: This master's current role is: LEADER
21:04:05.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.125531 29848 raft_consensus.cc:359] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:05.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.126039 29848 raft_consensus.cc:385] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:05.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.126210 29848 raft_consensus.cc:740] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 21371941629d4fd4b6222c61bce67340, State: Initialized, Role: FOLLOWER
21:04:05.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.126375 29952 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c16a40ef4eea44bf9ee93bded181a256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:05.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.126660 29952 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [sys.catalog]: This master's current role is: FOLLOWER
21:04:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.127208 29848 consensus_queue.cc:260] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.131673 29960 catalog_manager.cc:1357] Generated new cluster ID: 51f0d1f107744328bb1feb997b8b55af
21:04:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.131803 29960 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:04:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.135434 29848 sys_catalog.cc:565] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [sys.catalog]: configured and running, proceeding with master startup.
21:04:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.137279 29972 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.137585 29972 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [sys.catalog]: This master's current role is: FOLLOWER
21:04:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.139065 29811 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 0 FOLLOWER]: Advancing to term 1
21:04:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.133949 29691 master_runner.cc:186] Error getting master registration for 127.28.252.61:36963: OK, instance_id { permanent_uuid: "21371941629d4fd4b6222c61bce67340" instance_seqno: 1765400644319258 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:04:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.140971 29691 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.141198 29812 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c16a40ef4eea44bf9ee93bded181a256" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "21371941629d4fd4b6222c61bce67340"
21:04:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.142911 29813 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c16a40ef4eea44bf9ee93bded181a256" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "21371941629d4fd4b6222c61bce67340" is_pre_election: true
21:04:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.142925 29811 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 1 FOLLOWER]: Refusing update from remote peer c16a40ef4eea44bf9ee93bded181a256: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:05.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.144354 29938 consensus_queue.cc:1048] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:05.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.167652 29973 catalog_manager.cc:1269] Loaded cluster ID: 51f0d1f107744328bb1feb997b8b55af
21:04:05.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.167806 29973 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268: loading cluster ID for follower catalog manager: success
21:04:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.169621 29973 catalog_manager.cc:1586] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268: acquiring CA information for follower catalog manager: Not found: root CA entry not found
21:04:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.169816 29973 catalog_manager.cc:883] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
21:04:05.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.185339 29982 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c16a40ef4eea44bf9ee93bded181a256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:05.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.185420 29972 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c16a40ef4eea44bf9ee93bded181a256. Latest consensus state: current_term: 1 leader_uuid: "c16a40ef4eea44bf9ee93bded181a256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:05.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.186030 29972 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [sys.catalog]: This master's current role is: FOLLOWER
21:04:05.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.186077 29982 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [sys.catalog]: This master's current role is: FOLLOWER
21:04:05.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.212375 29990 catalog_manager.cc:1269] Loaded cluster ID: 51f0d1f107744328bb1feb997b8b55af
21:04:05.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.212517 29990 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340: loading cluster ID for follower catalog manager: success
21:04:05.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.214301 29990 catalog_manager.cc:1586] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340: acquiring CA information for follower catalog manager: Not found: root CA entry not found
21:04:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.214481 29990 catalog_manager.cc:883] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
21:04:05.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.221570 29938 sys_catalog.cc:455] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c16a40ef4eea44bf9ee93bded181a256" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:05.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.222054 29938 sys_catalog.cc:458] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256 [sys.catalog]: This master's current role is: LEADER
21:04:05.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.223392 29960 catalog_manager.cc:1380] Generated new certificate authority record
21:04:05.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.225844 29960 catalog_manager.cc:1514] Loading token signing keys...
21:04:05.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.245023 29960 catalog_manager.cc:6027] T 00000000000000000000000000000000 P c16a40ef4eea44bf9ee93bded181a256: Generated new TSK 0
21:04:05.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.246462 29960 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.260872 29680 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/wal
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/data
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/logs
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/data/info.pb
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.1:0
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.252.1
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.1
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:04:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.806658 29993 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.807063 29993 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.807237 29993 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:05.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.816675 29993 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.816869 29993 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:04:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.816946 29993 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:04:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.817008 29993 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:04:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.817144 29993 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.252.1
21:04:05.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.827690 29993 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/data
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/wal
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.1:0
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/data/info.pb
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.1
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.252.1
21:04:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/logs
21:04:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:04:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 20:43:25 UTC on 24a791456cd2
21:04:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9452
21:04:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.829756 29993 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:05.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.831703 29993 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.841969 29998 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:05.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.844825 29999 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:05.850836 30001 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:05.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.851447 29993 server_base.cc:1047] running on GCE node
21:04:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.852154 29993 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:05.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.853322 29993 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:05.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.854610 29993 hybrid_clock.cc:648] HybridClock initialized: now 1765400645854481 us; error 49 us; skew 500 ppm
21:04:05.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.854969 29993 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.857390 29993 webserver.cc:492] Webserver started at http://127.28.252.1:36541/ using document root <none> and password file <none>
21:04:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.858346 29993 fs_manager.cc:362] Metadata directory not provided
21:04:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.858502 29993 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:05.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.859038 29993 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:05.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.861573 29993 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/data/instance:
21:04:05.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63bf962d57384f75aea7b1e40362cd23"
21:04:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:05 on dist-test-slave-nl76"
21:04:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.862385 29993 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/wal/instance:
21:04:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63bf962d57384f75aea7b1e40362cd23"
21:04:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:05 on dist-test-slave-nl76"
21:04:05.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.868119 29993 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.000s sys 0.004s
21:04:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.871764 30007 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.873374 29993 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
21:04:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.873529 29993 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/data,/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/wal
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63bf962d57384f75aea7b1e40362cd23"
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:05 on dist-test-slave-nl76"
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.873715 29993 fs_report.cc:389] FS layout report
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/wal
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/wal
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/data/data
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.900022 29993 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.900914 29993 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.901221 29993 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.904588 29993 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.904726 29993 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:04:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.904852 29993 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:05.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.904997 29993 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:04:05.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.971172 29993 rpc_server.cc:307] RPC server started. Bound to: 127.28.252.1:34617
21:04:05.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.971246 30131 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.252.1:34617 every 8 connection(s)
21:04:05.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.972537 29993 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/data/info.pb
21:04:05.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.975610 30133 heartbeater.cc:344] Connected to a master server at 127.28.252.62:33507
21:04:05.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.975991 30133 heartbeater.cc:461] Registering TS with master...
21:04:05.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.976974 30133 heartbeater.cc:507] Master 127.28.252.62:33507 requested a full tablet report, sending...
21:04:05.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.978853 29680 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu as pid 29993
21:04:05.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.979035 29680 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-0/wal/instance
21:04:05.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.979432 29722 ts_manager.cc:194] Registered new tserver with Master: 63bf962d57384f75aea7b1e40362cd23 (127.28.252.1:34617)
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.983927 30132 heartbeater.cc:344] Connected to a master server at 127.28.252.60:39391
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.984109 30132 heartbeater.cc:461] Registering TS with master...
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.984483 29680 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/wal
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/data
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/logs
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/data/info.pb
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.2:0
21:04:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.252.2
21:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.2
21:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:04:05.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.984725 30132 heartbeater.cc:507] Master 127.28.252.60:39391 requested a full tablet report, sending...
21:04:05.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.987202 29884 ts_manager.cc:194] Registered new tserver with Master: 63bf962d57384f75aea7b1e40362cd23 (127.28.252.1:34617)
21:04:05.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.988696 30134 heartbeater.cc:344] Connected to a master server at 127.28.252.61:36963
21:04:05.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.988901 30134 heartbeater.cc:461] Registering TS with master...
21:04:05.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.989305 29884 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.252.1:54325
21:04:05.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.989543 30134 heartbeater.cc:507] Master 127.28.252.61:36963 requested a full tablet report, sending...
21:04:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:05.991483 29803 ts_manager.cc:194] Registered new tserver with Master: 63bf962d57384f75aea7b1e40362cd23 (127.28.252.1:34617)
21:04:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.175139 29973 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268: acquiring CA information for follower catalog manager: success
21:04:06.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.177826 29973 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:04:06.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.219285 29990 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340: acquiring CA information for follower catalog manager: success
21:04:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.222115 29990 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:04:06.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.349372 30137 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:06.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.349730 30137 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:06.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.349831 30137 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.359102 30137 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.359261 30137 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:04:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.359313 30137 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:04:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.359349 30137 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:04:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.359431 30137 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.252.2
21:04:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.369588 30137 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/data
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/wal
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.2:0
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/data/info.pb
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.2
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.252.2
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/logs
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 20:43:25 UTC on 24a791456cd2
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9452
21:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:06.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.371299 30137 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:06.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.373107 30137 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:06.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.383316 30143 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:06.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.383849 30142 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:06.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.384294 30145 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:06.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.384925 30137 server_base.cc:1047] running on GCE node
21:04:06.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.385625 30137 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:06.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.386684 30137 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:06.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.387987 30137 hybrid_clock.cc:648] HybridClock initialized: now 1765400646387892 us; error 55 us; skew 500 ppm
21:04:06.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.388383 30137 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:06.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.390872 30137 webserver.cc:492] Webserver started at http://127.28.252.2:39209/ using document root <none> and password file <none>
21:04:06.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.391701 30137 fs_manager.cc:362] Metadata directory not provided
21:04:06.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.391844 30137 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:06.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.392206 30137 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:06.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.394886 30137 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/data/instance:
21:04:06.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d689e65092aa42f88b5c3a861f4dc7bc"
21:04:06.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:06 on dist-test-slave-nl76"
21:04:06.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.395781 30137 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/wal/instance:
21:04:06.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d689e65092aa42f88b5c3a861f4dc7bc"
21:04:06.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:06 on dist-test-slave-nl76"
21:04:06.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.401805 30137 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
21:04:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.405900 30151 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.407567 30137 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.407763 30137 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/data,/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/wal
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d689e65092aa42f88b5c3a861f4dc7bc"
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:06 on dist-test-slave-nl76"
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.408010 30137 fs_report.cc:389] FS layout report
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/wal
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/wal
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/data/data
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.449828 30137 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.450883 30137 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.451220 30137 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:06.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.454744 30137 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.454879 30137 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:04:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.455034 30137 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.455138 30137 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:04:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.529927 30137 rpc_server.cc:307] RPC server started. Bound to: 127.28.252.2:34483
21:04:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.529975 30273 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.252.2:34483 every 8 connection(s)
21:04:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.531409 30137 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/data/info.pb
21:04:06.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.534819 30275 heartbeater.cc:344] Connected to a master server at 127.28.252.62:33507
21:04:06.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.535221 30275 heartbeater.cc:461] Registering TS with master...
21:04:06.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.536262 30275 heartbeater.cc:507] Master 127.28.252.62:33507 requested a full tablet report, sending...
21:04:06.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.537935 29680 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu as pid 30137
21:04:06.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.538128 29680 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-1/wal/instance
21:04:06.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.538427 29722 ts_manager.cc:194] Registered new tserver with Master: d689e65092aa42f88b5c3a861f4dc7bc (127.28.252.2:34483)
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.542897 30274 heartbeater.cc:344] Connected to a master server at 127.28.252.60:39391
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.543052 30274 heartbeater.cc:461] Registering TS with master...
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.543527 30274 heartbeater.cc:507] Master 127.28.252.60:39391 requested a full tablet report, sending...
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.543838 29680 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/wal
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/data
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/logs
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/data/info.pb
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.3:0
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.252.3
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.3
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.544539 29884 ts_manager.cc:194] Registered new tserver with Master: d689e65092aa42f88b5c3a861f4dc7bc (127.28.252.2:34483)
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.545313 29884 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.252.2:55495
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.546340 30276 heartbeater.cc:344] Connected to a master server at 127.28.252.61:36963
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.546479 30276 heartbeater.cc:461] Registering TS with master...
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.547298 30276 heartbeater.cc:507] Master 127.28.252.61:36963 requested a full tablet report, sending...
21:04:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.548477 29803 ts_manager.cc:194] Registered new tserver with Master: d689e65092aa42f88b5c3a861f4dc7bc (127.28.252.2:34483)
21:04:06.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.891005 30279 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:06.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.891425 30279 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:06.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.891583 30279 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.900228 30279 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.900377 30279 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:04:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.900431 30279 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:04:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.900467 30279 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:04:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.900555 30279 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.252.3
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.911125 30279 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.252.20:42411
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/data
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/wal
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.252.3:0
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/data/info.pb
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.252.3
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.252.62:33507,127.28.252.61:36963,127.28.252.60:39391
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.252.3
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/logs
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:04:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 20:43:25 UTC on 24a791456cd2
21:04:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9452
21:04:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:06.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.912873 30279 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.914846 30279 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.926116 30284 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.926255 30285 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:06.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:06.927376 30287 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:06.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.927671 30279 server_base.cc:1047] running on GCE node
21:04:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.928258 30279 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.929173 30279 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:06.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.930410 30279 hybrid_clock.cc:648] HybridClock initialized: now 1765400646930349 us; error 43 us; skew 500 ppm
21:04:06.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.930747 30279 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.933046 30279 webserver.cc:492] Webserver started at http://127.28.252.3:43821/ using document root <none> and password file <none>
21:04:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.933823 30279 fs_manager.cc:362] Metadata directory not provided
21:04:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.933924 30279 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.934221 30279 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.936806 30279 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/data/instance:
21:04:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2adffa9b6b6d48f08319c66402af52e6"
21:04:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:06 on dist-test-slave-nl76"
21:04:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.937551 30279 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/wal/instance:
21:04:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2adffa9b6b6d48f08319c66402af52e6"
21:04:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:06 on dist-test-slave-nl76"
21:04:06.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.943146 30279 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.002s
21:04:06.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.946723 30293 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:06.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.948417 30279 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
21:04:06.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.948608 30279 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/data,/tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/wal
21:04:06.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2adffa9b6b6d48f08319c66402af52e6"
21:04:06.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 21:04:06 on dist-test-slave-nl76"
21:04:06.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.948807 30279 fs_report.cc:389] FS layout report
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/wal
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/wal
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/data/data
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.969336 30279 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:06.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.970187 30279 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:06.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.970454 30279 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:06.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.974133 30279 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:06.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.974440 30279 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:04:06.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.974680 30279 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:06.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.974788 30279 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:04:06.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:06.992218 30132 heartbeater.cc:499] Master 127.28.252.60:39391 was elected leader, sending a full tablet report...
21:04:07.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.037087 30279 rpc_server.cc:307] RPC server started. Bound to: 127.28.252.3:36921
21:04:07.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.037163 30414 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.252.3:36921 every 8 connection(s)
21:04:07.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.038471 30279 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/data/info.pb
21:04:07.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.041079 30416 heartbeater.cc:344] Connected to a master server at 127.28.252.62:33507
21:04:07.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.041707 30416 heartbeater.cc:461] Registering TS with master...
21:04:07.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.043051 30416 heartbeater.cc:507] Master 127.28.252.62:33507 requested a full tablet report, sending...
21:04:07.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.045038 29722 ts_manager.cc:194] Registered new tserver with Master: 2adffa9b6b6d48f08319c66402af52e6 (127.28.252.3:36921)
21:04:07.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.045579 29680 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu as pid 30279
21:04:07.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.045747 29680 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskfRBwJn/test-tmp/mini-kudu-cluster6886381340807075263/ts-2/wal/instance
21:04:07.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.049636 30415 heartbeater.cc:344] Connected to a master server at 127.28.252.60:39391
21:04:07.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.049811 30415 heartbeater.cc:461] Registering TS with master...
21:04:07.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.050503 30415 heartbeater.cc:507] Master 127.28.252.60:39391 requested a full tablet report, sending...
21:04:07.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.051530 29884 ts_manager.cc:194] Registered new tserver with Master: 2adffa9b6b6d48f08319c66402af52e6 (127.28.252.3:36921)
21:04:07.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.051703 30417 heartbeater.cc:344] Connected to a master server at 127.28.252.61:36963
21:04:07.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.051884 30417 heartbeater.cc:461] Registering TS with master...
21:04:07.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.052479 30417 heartbeater.cc:507] Master 127.28.252.61:36963 requested a full tablet report, sending...
21:04:07.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.052729 29884 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.252.3:51309
21:04:07.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.053617 29802 ts_manager.cc:194] Registered new tserver with Master: 2adffa9b6b6d48f08319c66402af52e6 (127.28.252.3:36921)
21:04:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.057027 29680 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:04:07.058 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:04:07.059 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:04:07.060 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.252.62"
port: 33507
}
bound_http_address {
host: "127.28.252.62"
port: 46835
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.252.61"
port: 36963
}
bound_http_address {
host: "127.28.252.61"
port: 35771
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.252.60"
port: 39391
}
bound_http_address {
host: "127.28.252.60"
port: 42755
}
}
}
21:04:07.060 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:04:07.061 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.28.252.1"
port: 34617
}
bound_http_address {
host: "127.28.252.1"
port: 36541
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.28.252.2"
port: 34483
}
bound_http_address {
host: "127.28.252.2"
port: 39209
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.28.252.3"
port: 36921
}
bound_http_address {
host: "127.28.252.3"
port: 43821
}
}
}
21:04:07.061 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:04:07.063 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:04:07.064 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.252.61:36963(127.28.252.61:36963)] connecting to peer
21:04:07.065 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.252.62:33507(127.28.252.62:33507)] connecting to peer
21:04:07.066 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:07.066 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.252.61:36963(127.28.252.61:36963)] Successfully connected to peer
21:04:07.068 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.28.252.61
21:04:07.072 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.252.60:39391(127.28.252.60:39391)] Successfully connected to peer
21:04:07.068 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.252.62:33507(127.28.252.62:33507)] Successfully connected to peer
21:04:07.075 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.28.252.62
21:04:07.081 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.28.252.60
21:04:07.085 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x3e792b02, L:/127.0.0.1:46260 - R:/127.28.252.62:33507] using SASL/PLAIN
21:04:07.087 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4d12f78e, L:/127.0.0.1:46122 - R:/127.28.252.61:36963] using SASL/PLAIN
21:04:07.093 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3d10ae04, L:/127.0.0.1:51236 - R:/127.28.252.60:39391] using SASL/PLAIN
21:04:07.096 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:04:07.096 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=32), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [23ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [28ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [31ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: OK, deferred=null) after lookup
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.098250 29883 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51236:
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
21:04:07.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:07.100198 29883 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:07.119 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 6948525bb3a649e7bdefe5fbd06341e6
21:04:07.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.164469 30066 tablet_service.cc:1505] Processing CreateTablet for tablet 1fea82a6305d458293803989630fc268 (DEFAULT_TABLE table=txn_manager_fallback [id=6948525bb3a649e7bdefe5fbd06341e6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:04:07.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.166182 30066 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1fea82a6305d458293803989630fc268. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:07.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.165599 30346 tablet_service.cc:1505] Processing CreateTablet for tablet 1fea82a6305d458293803989630fc268 (DEFAULT_TABLE table=txn_manager_fallback [id=6948525bb3a649e7bdefe5fbd06341e6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:04:07.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.167281 30346 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1fea82a6305d458293803989630fc268. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:07.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.167219 30347 tablet_service.cc:1505] Processing CreateTablet for tablet 90aac9cc04484a25a2a54dc2674c5dee (DEFAULT_TABLE table=txn_manager_fallback [id=6948525bb3a649e7bdefe5fbd06341e6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:04:07.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.167984 30347 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 90aac9cc04484a25a2a54dc2674c5dee. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:07.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.170596 30067 tablet_service.cc:1505] Processing CreateTablet for tablet 90aac9cc04484a25a2a54dc2674c5dee (DEFAULT_TABLE table=txn_manager_fallback [id=6948525bb3a649e7bdefe5fbd06341e6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:04:07.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.171154 30067 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 90aac9cc04484a25a2a54dc2674c5dee. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.170982 30209 tablet_service.cc:1505] Processing CreateTablet for tablet 90aac9cc04484a25a2a54dc2674c5dee (DEFAULT_TABLE table=txn_manager_fallback [id=6948525bb3a649e7bdefe5fbd06341e6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:04:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.172289 30209 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 90aac9cc04484a25a2a54dc2674c5dee. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.170060 30208 tablet_service.cc:1505] Processing CreateTablet for tablet 1fea82a6305d458293803989630fc268 (DEFAULT_TABLE table=txn_manager_fallback [id=6948525bb3a649e7bdefe5fbd06341e6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:04:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.175199 30208 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1fea82a6305d458293803989630fc268. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.188393 30440 tablet_bootstrap.cc:492] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23: Bootstrap starting.
21:04:07.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.193763 30440 tablet_bootstrap.cc:654] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23: Neither blocks nor log segments found. Creating new log.
21:04:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.195600 30440 log.cc:826] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23: Log is configured to *not* fsync() on all Append() calls
21:04:07.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.197108 30441 tablet_bootstrap.cc:492] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6: Bootstrap starting.
21:04:07.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.199759 30440 tablet_bootstrap.cc:492] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23: No bootstrap required, opened a new log
21:04:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.200095 30440 ts_tablet_manager.cc:1403] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23: Time spent bootstrapping tablet: real 0.012s user 0.008s sys 0.001s
21:04:07.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.200820 30442 tablet_bootstrap.cc:492] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc: Bootstrap starting.
21:04:07.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.201406 30441 tablet_bootstrap.cc:654] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6: Neither blocks nor log segments found. Creating new log.
21:04:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.204813 30442 tablet_bootstrap.cc:654] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc: Neither blocks nor log segments found. Creating new log.
21:04:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.204789 30441 log.cc:826] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6: Log is configured to *not* fsync() on all Append() calls
21:04:07.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.206480 30442 log.cc:826] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc: Log is configured to *not* fsync() on all Append() calls
21:04:07.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.211339 30440 raft_consensus.cc:359] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } }
21:04:07.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.211935 30440 raft_consensus.cc:385] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:07.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.212055 30440 raft_consensus.cc:740] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 63bf962d57384f75aea7b1e40362cd23, State: Initialized, Role: FOLLOWER
21:04:07.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.213146 30440 consensus_queue.cc:260] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } }
21:04:07.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.213760 30442 tablet_bootstrap.cc:492] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc: No bootstrap required, opened a new log
21:04:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.214185 30442 ts_tablet_manager.cc:1403] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc: Time spent bootstrapping tablet: real 0.015s user 0.008s sys 0.004s
21:04:07.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.220932 30440 ts_tablet_manager.cc:1434] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23: Time spent starting tablet: real 0.021s user 0.013s sys 0.006s
21:04:07.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.221858 30440 tablet_bootstrap.cc:492] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23: Bootstrap starting.
21:04:07.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.221941 30441 tablet_bootstrap.cc:492] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6: No bootstrap required, opened a new log
21:04:07.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.222252 30441 ts_tablet_manager.cc:1403] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6: Time spent bootstrapping tablet: real 0.025s user 0.013s sys 0.009s
21:04:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.223554 30442 raft_consensus.cc:359] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } }
21:04:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.224009 30440 tablet_bootstrap.cc:654] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23: Neither blocks nor log segments found. Creating new log.
21:04:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.224262 30442 raft_consensus.cc:385] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.224481 30442 raft_consensus.cc:740] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d689e65092aa42f88b5c3a861f4dc7bc, State: Initialized, Role: FOLLOWER
21:04:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.225445 30442 consensus_queue.cc:260] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } }
21:04:07.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:07.230130 30135 tablet.cc:2378] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.232831 30441 raft_consensus.cc:359] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.233551 30441 raft_consensus.cc:385] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.233704 30441 raft_consensus.cc:740] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2adffa9b6b6d48f08319c66402af52e6, State: Initialized, Role: FOLLOWER
21:04:07.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.234889 30441 consensus_queue.cc:260] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.236562 30415 heartbeater.cc:499] Master 127.28.252.60:39391 was elected leader, sending a full tablet report...
21:04:07.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.237586 30441 ts_tablet_manager.cc:1434] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6: Time spent starting tablet: real 0.015s user 0.011s sys 0.003s
21:04:07.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.237620 30440 tablet_bootstrap.cc:492] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23: No bootstrap required, opened a new log
21:04:07.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.237951 30440 ts_tablet_manager.cc:1403] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23: Time spent bootstrapping tablet: real 0.016s user 0.005s sys 0.008s
21:04:07.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.238096 30441 tablet_bootstrap.cc:492] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6: Bootstrap starting.
21:04:07.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.238955 30440 raft_consensus.cc:359] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.239183 30440 raft_consensus.cc:385] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:07.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.239269 30440 raft_consensus.cc:740] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 63bf962d57384f75aea7b1e40362cd23, State: Initialized, Role: FOLLOWER
21:04:07.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.239511 30440 consensus_queue.cc:260] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.240056 30441 tablet_bootstrap.cc:654] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6: Neither blocks nor log segments found. Creating new log.
21:04:07.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.241649 30440 ts_tablet_manager.cc:1434] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
21:04:07.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.252148 30274 heartbeater.cc:499] Master 127.28.252.60:39391 was elected leader, sending a full tablet report...
21:04:07.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.252539 30442 ts_tablet_manager.cc:1434] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc: Time spent starting tablet: real 0.038s user 0.017s sys 0.011s
21:04:07.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.252830 30441 tablet_bootstrap.cc:492] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6: No bootstrap required, opened a new log
21:04:07.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.254228 30441 ts_tablet_manager.cc:1403] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6: Time spent bootstrapping tablet: real 0.016s user 0.007s sys 0.007s
21:04:07.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.254633 30442 tablet_bootstrap.cc:492] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc: Bootstrap starting.
21:04:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.255322 30441 raft_consensus.cc:359] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } }
21:04:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.255548 30441 raft_consensus.cc:385] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.255647 30441 raft_consensus.cc:740] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2adffa9b6b6d48f08319c66402af52e6, State: Initialized, Role: FOLLOWER
21:04:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.256006 30441 consensus_queue.cc:260] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } }
21:04:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.257774 30442 tablet_bootstrap.cc:654] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc: Neither blocks nor log segments found. Creating new log.
21:04:07.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.258509 30441 ts_tablet_manager.cc:1434] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
21:04:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.260886 30442 tablet_bootstrap.cc:492] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc: No bootstrap required, opened a new log
21:04:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.261075 30442 ts_tablet_manager.cc:1403] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.003s
21:04:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.262303 30442 raft_consensus.cc:359] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.262494 30442 raft_consensus.cc:385] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:07.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.262609 30442 raft_consensus.cc:740] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d689e65092aa42f88b5c3a861f4dc7bc, State: Initialized, Role: FOLLOWER
21:04:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.262856 30442 consensus_queue.cc:260] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.263455 30442 ts_tablet_manager.cc:1434] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
21:04:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:07.285334 30277 tablet.cc:2378] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.290509 30449 raft_consensus.cc:493] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.290818 30449 raft_consensus.cc:515] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:07.291955 30418 tablet.cc:2378] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.293076 30449 leader_election.cc:290] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 63bf962d57384f75aea7b1e40362cd23 (127.28.252.1:34617), d689e65092aa42f88b5c3a861f4dc7bc (127.28.252.2:34483)
21:04:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.300506 30229 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "1fea82a6305d458293803989630fc268" candidate_uuid: "2adffa9b6b6d48f08319c66402af52e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" is_pre_election: true
21:04:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.300704 30087 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "1fea82a6305d458293803989630fc268" candidate_uuid: "2adffa9b6b6d48f08319c66402af52e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "63bf962d57384f75aea7b1e40362cd23" is_pre_election: true
21:04:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.301141 30087 raft_consensus.cc:2468] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2adffa9b6b6d48f08319c66402af52e6 in term 0.
21:04:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.301148 30229 raft_consensus.cc:2468] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2adffa9b6b6d48f08319c66402af52e6 in term 0.
21:04:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.302003 30297 leader_election.cc:304] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 2adffa9b6b6d48f08319c66402af52e6, 63bf962d57384f75aea7b1e40362cd23, d689e65092aa42f88b5c3a861f4dc7bc; no voters:
21:04:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.302441 30449 raft_consensus.cc:2804] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.302639 30449 raft_consensus.cc:493] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.302749 30449 raft_consensus.cc:3060] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Advancing to term 1
21:04:07.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.304945 30449 raft_consensus.cc:515] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.305738 30449 leader_election.cc:290] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [CANDIDATE]: Term 1 election: Requested vote from peers 63bf962d57384f75aea7b1e40362cd23 (127.28.252.1:34617), d689e65092aa42f88b5c3a861f4dc7bc (127.28.252.2:34483)
21:04:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.306336 30087 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "1fea82a6305d458293803989630fc268" candidate_uuid: "2adffa9b6b6d48f08319c66402af52e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "63bf962d57384f75aea7b1e40362cd23"
21:04:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.306499 30229 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "1fea82a6305d458293803989630fc268" candidate_uuid: "2adffa9b6b6d48f08319c66402af52e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d689e65092aa42f88b5c3a861f4dc7bc"
21:04:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.306568 30087 raft_consensus.cc:3060] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Advancing to term 1
21:04:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.306780 30229 raft_consensus.cc:3060] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Advancing to term 1
21:04:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.309098 30087 raft_consensus.cc:2468] T 1fea82a6305d458293803989630fc268 P 63bf962d57384f75aea7b1e40362cd23 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2adffa9b6b6d48f08319c66402af52e6 in term 1.
21:04:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.309166 30229 raft_consensus.cc:2468] T 1fea82a6305d458293803989630fc268 P d689e65092aa42f88b5c3a861f4dc7bc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2adffa9b6b6d48f08319c66402af52e6 in term 1.
21:04:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.309650 30294 leader_election.cc:304] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2adffa9b6b6d48f08319c66402af52e6, 63bf962d57384f75aea7b1e40362cd23; no voters:
21:04:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.309994 30449 raft_consensus.cc:2804] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [term 1 FOLLOWER]: Leader election won for term 1
21:04:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.310459 30449 raft_consensus.cc:697] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [term 1 LEADER]: Becoming Leader. State: Replica: 2adffa9b6b6d48f08319c66402af52e6, State: Running, Role: LEADER
21:04:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.311074 30449 consensus_queue.cc:237] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.316545 29883 catalog_manager.cc:5654] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2adffa9b6b6d48f08319c66402af52e6 (127.28.252.3). New cstate: current_term: 1 leader_uuid: "2adffa9b6b6d48f08319c66402af52e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } health_report { overall_health: HEALTHY } } }
21:04:07.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.416462 30449 raft_consensus.cc:493] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:07.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.416692 30449 raft_consensus.cc:515] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } }
21:04:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.417374 30449 leader_election.cc:290] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 63bf962d57384f75aea7b1e40362cd23 (127.28.252.1:34617), d689e65092aa42f88b5c3a861f4dc7bc (127.28.252.2:34483)
21:04:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.417953 30087 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "90aac9cc04484a25a2a54dc2674c5dee" candidate_uuid: "2adffa9b6b6d48f08319c66402af52e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "63bf962d57384f75aea7b1e40362cd23" is_pre_election: true
21:04:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.417958 30229 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "90aac9cc04484a25a2a54dc2674c5dee" candidate_uuid: "2adffa9b6b6d48f08319c66402af52e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" is_pre_election: true
21:04:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.418191 30229 raft_consensus.cc:2468] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2adffa9b6b6d48f08319c66402af52e6 in term 0.
21:04:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.418207 30087 raft_consensus.cc:2468] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2adffa9b6b6d48f08319c66402af52e6 in term 0.
21:04:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.418730 30297 leader_election.cc:304] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 2adffa9b6b6d48f08319c66402af52e6, 63bf962d57384f75aea7b1e40362cd23, d689e65092aa42f88b5c3a861f4dc7bc; no voters:
21:04:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.419090 30449 raft_consensus.cc:2804] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.419209 30449 raft_consensus.cc:493] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.419304 30449 raft_consensus.cc:3060] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Advancing to term 1
21:04:07.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.421125 30449 raft_consensus.cc:515] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } }
21:04:07.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.421722 30449 leader_election.cc:290] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [CANDIDATE]: Term 1 election: Requested vote from peers 63bf962d57384f75aea7b1e40362cd23 (127.28.252.1:34617), d689e65092aa42f88b5c3a861f4dc7bc (127.28.252.2:34483)
21:04:07.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.422222 30087 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "90aac9cc04484a25a2a54dc2674c5dee" candidate_uuid: "2adffa9b6b6d48f08319c66402af52e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "63bf962d57384f75aea7b1e40362cd23"
21:04:07.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.422268 30229 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "90aac9cc04484a25a2a54dc2674c5dee" candidate_uuid: "2adffa9b6b6d48f08319c66402af52e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d689e65092aa42f88b5c3a861f4dc7bc"
21:04:07.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.422465 30087 raft_consensus.cc:3060] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Advancing to term 1
21:04:07.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.422499 30229 raft_consensus.cc:3060] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Advancing to term 1
21:04:07.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.424535 30229 raft_consensus.cc:2468] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2adffa9b6b6d48f08319c66402af52e6 in term 1.
21:04:07.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.424517 30087 raft_consensus.cc:2468] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2adffa9b6b6d48f08319c66402af52e6 in term 1.
21:04:07.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.425040 30294 leader_election.cc:304] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 2adffa9b6b6d48f08319c66402af52e6, 63bf962d57384f75aea7b1e40362cd23, d689e65092aa42f88b5c3a861f4dc7bc; no voters:
21:04:07.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.425381 30449 raft_consensus.cc:2804] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [term 1 FOLLOWER]: Leader election won for term 1
21:04:07.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.425571 30449 raft_consensus.cc:697] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [term 1 LEADER]: Becoming Leader. State: Replica: 2adffa9b6b6d48f08319c66402af52e6, State: Running, Role: LEADER
21:04:07.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.426157 30449 consensus_queue.cc:237] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } }
21:04:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.429086 29883 catalog_manager.cc:5654] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2adffa9b6b6d48f08319c66402af52e6 (127.28.252.3). New cstate: current_term: 1 leader_uuid: "2adffa9b6b6d48f08319c66402af52e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } health_report { overall_health: UNKNOWN } } }
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.672919 29883 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51240:
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:04:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:07.673981 29883 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:07.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.686252 30066 tablet_service.cc:1505] Processing CreateTablet for tablet 43106605761a4cd28b24398f134279fb (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b558bf9e18f545d29356b296b36e702a]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:04:07.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.687505 30066 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 43106605761a4cd28b24398f134279fb. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.687500 30346 tablet_service.cc:1505] Processing CreateTablet for tablet 43106605761a4cd28b24398f134279fb (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b558bf9e18f545d29356b296b36e702a]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:04:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.688378 30346 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 43106605761a4cd28b24398f134279fb. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:07.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.686317 30209 tablet_service.cc:1505] Processing CreateTablet for tablet 43106605761a4cd28b24398f134279fb (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b558bf9e18f545d29356b296b36e702a]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:04:07.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.692198 30209 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 43106605761a4cd28b24398f134279fb. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.694720 30440 tablet_bootstrap.cc:492] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: Bootstrap starting.
21:04:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.695003 30441 tablet_bootstrap.cc:492] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6: Bootstrap starting.
21:04:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.696426 30441 tablet_bootstrap.cc:654] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6: Neither blocks nor log segments found. Creating new log.
21:04:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.696587 30440 tablet_bootstrap.cc:654] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: Neither blocks nor log segments found. Creating new log.
21:04:07.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.698036 30442 tablet_bootstrap.cc:492] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc: Bootstrap starting.
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.698892 30441 tablet_bootstrap.cc:492] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6: No bootstrap required, opened a new log
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.699044 30441 ts_tablet_manager.cc:1403] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.001s
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.699363 30440 tablet_bootstrap.cc:492] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: No bootstrap required, opened a new log
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.699545 30440 ts_tablet_manager.cc:1403] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.001s
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.699637 30441 raft_consensus.cc:359] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.699795 30441 raft_consensus.cc:385] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.699841 30441 raft_consensus.cc:740] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2adffa9b6b6d48f08319c66402af52e6, State: Initialized, Role: FOLLOWER
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.700016 30442 tablet_bootstrap.cc:654] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc: Neither blocks nor log segments found. Creating new log.
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.700055 30441 consensus_queue.cc:260] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.700572 30441 ts_tablet_manager.cc:1434] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.700500 30449 tablet_replica.cc:442] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } }
21:04:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.700740 30449 tablet_replica.cc:445] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:07.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.701901 30440 raft_consensus.cc:359] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.702136 30440 raft_consensus.cc:385] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:07.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.702198 30442 tablet_bootstrap.cc:492] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc: No bootstrap required, opened a new log
21:04:07.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.702231 30440 raft_consensus.cc:740] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 63bf962d57384f75aea7b1e40362cd23, State: Initialized, Role: FOLLOWER
21:04:07.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.702374 30442 ts_tablet_manager.cc:1403] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
21:04:07.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.703174 30442 raft_consensus.cc:359] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.703158 30440 consensus_queue.cc:260] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.703361 30442 raft_consensus.cc:385] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:07.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.703437 30442 raft_consensus.cc:740] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d689e65092aa42f88b5c3a861f4dc7bc, State: Initialized, Role: FOLLOWER
21:04:07.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.703787 30440 ts_tablet_manager.cc:1434] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
21:04:07.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.703738 30442 consensus_queue.cc:260] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.703699 30446 tablet_replica.cc:442] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } }
21:04:07.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.704051 30446 tablet_replica.cc:445] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:07.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.704200 30447 tablet_replica.cc:442] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } }
21:04:07.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.704468 30447 tablet_replica.cc:445] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.708231 30442 ts_tablet_manager.cc:1434] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
21:04:07.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.726616 30449 consensus_queue.cc:1048] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:07.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.741323 30449 consensus_queue.cc:1048] T 1fea82a6305d458293803989630fc268 P 2adffa9b6b6d48f08319c66402af52e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.818718 30446 raft_consensus.cc:493] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.818969 30446 raft_consensus.cc:515] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.820613 30446 leader_election.cc:290] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d689e65092aa42f88b5c3a861f4dc7bc (127.28.252.2:34483), 2adffa9b6b6d48f08319c66402af52e6 (127.28.252.3:36921)
21:04:07.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.828562 30229 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "43106605761a4cd28b24398f134279fb" candidate_uuid: "63bf962d57384f75aea7b1e40362cd23" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" is_pre_election: true
21:04:07.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.828661 30370 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "43106605761a4cd28b24398f134279fb" candidate_uuid: "63bf962d57384f75aea7b1e40362cd23" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2adffa9b6b6d48f08319c66402af52e6" is_pre_election: true
21:04:07.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.828855 30229 raft_consensus.cc:2468] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 63bf962d57384f75aea7b1e40362cd23 in term 0.
21:04:07.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.828994 30370 raft_consensus.cc:2468] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 63bf962d57384f75aea7b1e40362cd23 in term 0.
21:04:07.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.829408 30011 leader_election.cc:304] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 63bf962d57384f75aea7b1e40362cd23, d689e65092aa42f88b5c3a861f4dc7bc; no voters:
21:04:07.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.829807 30446 raft_consensus.cc:2804] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.829941 30446 raft_consensus.cc:493] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.830055 30446 raft_consensus.cc:3060] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [term 0 FOLLOWER]: Advancing to term 1
21:04:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.832211 30446 raft_consensus.cc:515] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.832902 30446 leader_election.cc:290] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [CANDIDATE]: Term 1 election: Requested vote from peers d689e65092aa42f88b5c3a861f4dc7bc (127.28.252.2:34483), 2adffa9b6b6d48f08319c66402af52e6 (127.28.252.3:36921)
21:04:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.833608 30229 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "43106605761a4cd28b24398f134279fb" candidate_uuid: "63bf962d57384f75aea7b1e40362cd23" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d689e65092aa42f88b5c3a861f4dc7bc"
21:04:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.833654 30370 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "43106605761a4cd28b24398f134279fb" candidate_uuid: "63bf962d57384f75aea7b1e40362cd23" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2adffa9b6b6d48f08319c66402af52e6"
21:04:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.833853 30229 raft_consensus.cc:3060] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc [term 0 FOLLOWER]: Advancing to term 1
21:04:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.833853 30370 raft_consensus.cc:3060] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6 [term 0 FOLLOWER]: Advancing to term 1
21:04:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.836843 30370 raft_consensus.cc:2468] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 63bf962d57384f75aea7b1e40362cd23 in term 1.
21:04:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.836843 30229 raft_consensus.cc:2468] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 63bf962d57384f75aea7b1e40362cd23 in term 1.
21:04:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.837386 30011 leader_election.cc:304] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 63bf962d57384f75aea7b1e40362cd23, d689e65092aa42f88b5c3a861f4dc7bc; no voters:
21:04:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.837798 30446 raft_consensus.cc:2804] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [term 1 FOLLOWER]: Leader election won for term 1
21:04:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.838289 30446 raft_consensus.cc:697] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [term 1 LEADER]: Becoming Leader. State: Replica: 63bf962d57384f75aea7b1e40362cd23, State: Running, Role: LEADER
21:04:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.838914 30446 consensus_queue.cc:237] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } }
21:04:07.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.842483 30481 tablet_replica.cc:442] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: TxnStatusTablet state changed. Reason: New leader 63bf962d57384f75aea7b1e40362cd23. Latest consensus state: current_term: 1 leader_uuid: "63bf962d57384f75aea7b1e40362cd23" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } }
21:04:07.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.842841 30481 tablet_replica.cc:445] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: This TxnStatusTablet replica's current role is: LEADER
21:04:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.843559 30483 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
21:04:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.844313 30483 txn_status_manager.cc:930] Loading transaction status metadata into memory...
21:04:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.844748 29884 catalog_manager.cc:5654] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 reported cstate change: term changed from 0 to 1, leader changed from <none> to 63bf962d57384f75aea7b1e40362cd23 (127.28.252.1). New cstate: current_term: 1 leader_uuid: "63bf962d57384f75aea7b1e40362cd23" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } health_report { overall_health: UNKNOWN } } }
21:04:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.881930 30229 raft_consensus.cc:1275] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc [term 1 FOLLOWER]: Refusing update from remote peer 63bf962d57384f75aea7b1e40362cd23: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.882301 30370 raft_consensus.cc:1275] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6 [term 1 FOLLOWER]: Refusing update from remote peer 63bf962d57384f75aea7b1e40362cd23: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.882710 30446 consensus_queue.cc:1048] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.883212 30481 consensus_queue.cc:1048] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.891923 30447 tablet_replica.cc:442] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc: TxnStatusTablet state changed. Reason: New leader 63bf962d57384f75aea7b1e40362cd23. Latest consensus state: current_term: 1 leader_uuid: "63bf962d57384f75aea7b1e40362cd23" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } }
21:04:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.892270 30447 tablet_replica.cc:445] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.896884 30473 mvcc.cc:204] Tried to move back new op lower bound from 7231081053715222528 to 7231081053553197056. Current Snapshot: MvccSnapshot[applied={T|T < 7231081053715222528}]
21:04:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.898433 30481 tablet_replica.cc:442] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "63bf962d57384f75aea7b1e40362cd23" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } }
21:04:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.898694 30481 tablet_replica.cc:445] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: This TxnStatusTablet replica's current role is: LEADER
21:04:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.900132 30447 tablet_replica.cc:442] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "63bf962d57384f75aea7b1e40362cd23" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } }
21:04:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.900389 30447 tablet_replica.cc:445] T 43106605761a4cd28b24398f134279fb P d689e65092aa42f88b5c3a861f4dc7bc: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:07.905 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
21:04:07.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.908293 30452 tablet_replica.cc:442] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6: TxnStatusTablet state changed. Reason: New leader 63bf962d57384f75aea7b1e40362cd23. Latest consensus state: current_term: 1 leader_uuid: "63bf962d57384f75aea7b1e40362cd23" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } }
21:04:07.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.908576 30452 tablet_replica.cc:445] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:07.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.909399 30446 tablet_replica.cc:442] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "63bf962d57384f75aea7b1e40362cd23" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } }
21:04:07.909 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 6948525bb3a649e7bdefe5fbd06341e6
21:04:07.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.909669 30446 tablet_replica.cc:445] T 43106605761a4cd28b24398f134279fb P 63bf962d57384f75aea7b1e40362cd23: This TxnStatusTablet replica's current role is: LEADER
21:04:07.912 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 90aac9cc04484a25a2a54dc2674c5dee for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
21:04:07.912 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1fea82a6305d458293803989630fc268 for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
21:04:07.912 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=90aac9cc04484a25a2a54dc2674c5dee@[2adffa9b6b6d48f08319c66402af52e6(127.28.252.3:36921)[L],63bf962d57384f75aea7b1e40362cd23(127.28.252.1:34617),d689e65092aa42f88b5c3a861f4dc7bc(127.28.252.2:34483)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=1fea82a6305d458293803989630fc268@[2adffa9b6b6d48f08319c66402af52e6(127.28.252.3:36921)[L],63bf962d57384f75aea7b1e40362cd23(127.28.252.1:34617),d689e65092aa42f88b5c3a861f4dc7bc(127.28.252.2:34483)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
21:04:07.912 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.28.252.60:39391, [2ms] Sub RPC GetTableLocations: received response from server master-127.28.252.60:39391: OK, deferred=null) after lookup
21:04:07.913 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2adffa9b6b6d48f08319c66402af52e6(127.28.252.3:36921)] connecting to peer
21:04:07.914 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 2adffa9b6b6d48f08319c66402af52e6(127.28.252.3:36921)] Successfully connected to peer
21:04:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.918839 30452 tablet_replica.cc:442] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "63bf962d57384f75aea7b1e40362cd23" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 } } peers { permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 } } peers { permanent_uuid: "2adffa9b6b6d48f08319c66402af52e6" member_type: VOTER last_known_addr { host: "127.28.252.3" port: 36921 } } }
21:04:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.919080 30452 tablet_replica.cc:445] T 43106605761a4cd28b24398f134279fb P 2adffa9b6b6d48f08319c66402af52e6: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:07.933 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf8f57e04, L:/127.0.0.1:56688 - R:/127.28.252.3:36921] using TOKEN/null
21:04:07.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.964890 30229 raft_consensus.cc:1275] T 90aac9cc04484a25a2a54dc2674c5dee P d689e65092aa42f88b5c3a861f4dc7bc [term 1 FOLLOWER]: Refusing update from remote peer 2adffa9b6b6d48f08319c66402af52e6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:07.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.965301 30087 raft_consensus.cc:1275] T 90aac9cc04484a25a2a54dc2674c5dee P 63bf962d57384f75aea7b1e40362cd23 [term 1 FOLLOWER]: Refusing update from remote peer 2adffa9b6b6d48f08319c66402af52e6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:07.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.965880 30449 consensus_queue.cc:1048] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "63bf962d57384f75aea7b1e40362cd23" member_type: VOTER last_known_addr { host: "127.28.252.1" port: 34617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:07.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.966361 30452 consensus_queue.cc:1048] T 90aac9cc04484a25a2a54dc2674c5dee P 2adffa9b6b6d48f08319c66402af52e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d689e65092aa42f88b5c3a861f4dc7bc" member_type: VOTER last_known_addr { host: "127.28.252.2" port: 34483 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:07.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:07.988320 30472 mvcc.cc:204] Tried to move back new op lower bound from 7231081054057824256 to 7231081051860119552. Current Snapshot: MvccSnapshot[applied={T|T < 7231081054057824256}]
21:04:08.008 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.252.60:39391
21:04:08.008 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:04:08.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.009240 29680 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu pid 29850
21:04:08.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.010811 29924 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:08.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.176162 29680 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu with pid 29850
21:04:08.199 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.252.60:39391(127.28.252.60:39391)] handling channelInactive
21:04:08.200 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state READY due to: connection closed
21:04:08.201 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:04:08.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.201040 29706 connection.cc:537] server connection from 127.0.0.1:46250 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:08.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.201300 29790 connection.cc:537] server connection from 127.0.0.1:46118 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:08.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.319078 30415 heartbeater.cc:646] Failed to heartbeat to 127.28.252.60:39391 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:08.428 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.429 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.28.252.60:39391(127.28.252.60:39391) for tablet Kudu Master: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.445 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.446 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.447 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.465 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.466 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.467 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.477465 30500 raft_consensus.cc:493] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c16a40ef4eea44bf9ee93bded181a256)
21:04:08.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.477711 30500 raft_consensus.cc:515] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:08.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.478420 30501 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c16a40ef4eea44bf9ee93bded181a256)
21:04:08.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.478660 30501 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:08.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.479317 30500 leader_election.cc:290] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6b7ebd5a66a24240b96752e63ff38268 (127.28.252.62:33507), c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391)
21:04:08.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.479878 29731 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "21371941629d4fd4b6222c61bce67340" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "6b7ebd5a66a24240b96752e63ff38268" is_pre_election: true
21:04:08.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.480125 29731 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 21371941629d4fd4b6222c61bce67340 in term 1.
21:04:08.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.480130 30501 leader_election.cc:290] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 21371941629d4fd4b6222c61bce67340 (127.28.252.61:36963), c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391)
21:04:08.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.480695 29811 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6b7ebd5a66a24240b96752e63ff38268" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "21371941629d4fd4b6222c61bce67340" is_pre_election: true
21:04:08.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.480652 29788 leader_election.cc:304] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 21371941629d4fd4b6222c61bce67340, 6b7ebd5a66a24240b96752e63ff38268; no voters:
21:04:08.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.480908 29811 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b7ebd5a66a24240b96752e63ff38268 in term 1.
21:04:08.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.481072 30500 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:04:08.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.481205 30500 raft_consensus.cc:493] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 1 FOLLOWER]: Starting leader election (detected failure of leader c16a40ef4eea44bf9ee93bded181a256)
21:04:08.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.481314 30500 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 1 FOLLOWER]: Advancing to term 2
21:04:08.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.482648 29709 leader_election.cc:304] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 21371941629d4fd4b6222c61bce67340, 6b7ebd5a66a24240b96752e63ff38268; no voters:
21:04:08.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.483073 30501 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:04:08.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.483078 29790 leader_election.cc:336] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391): Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:08.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.483333 30501 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 1 FOLLOWER]: Starting leader election (detected failure of leader c16a40ef4eea44bf9ee93bded181a256)
21:04:08.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.483554 30501 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 1 FOLLOWER]: Advancing to term 2
21:04:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.484467 30500 raft_consensus.cc:515] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.484838 29709 leader_election.cc:336] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391): Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:08.485 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.485957 29731 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "21371941629d4fd4b6222c61bce67340" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "6b7ebd5a66a24240b96752e63ff38268"
21:04:08.487 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.486943 30500 leader_election.cc:290] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [CANDIDATE]: Term 2 election: Requested vote from peers 6b7ebd5a66a24240b96752e63ff38268 (127.28.252.62:33507), c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391)
21:04:08.487 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.487 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.488034 29790 leader_election.cc:336] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391): Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:08.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.488137 30501 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:08.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.488566 29731 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 21371941629d4fd4b6222c61bce67340 in current term 2: Already voted for candidate 6b7ebd5a66a24240b96752e63ff38268 in this term.
21:04:08.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.488778 30501 leader_election.cc:290] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 2 election: Requested vote from peers 21371941629d4fd4b6222c61bce67340 (127.28.252.61:36963), c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391)
21:04:08.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.489149 29811 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6b7ebd5a66a24240b96752e63ff38268" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "21371941629d4fd4b6222c61bce67340"
21:04:08.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.489475 29811 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 6b7ebd5a66a24240b96752e63ff38268 in current term 2: Already voted for candidate 21371941629d4fd4b6222c61bce67340 in this term.
21:04:08.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.489652 29788 leader_election.cc:304] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 21371941629d4fd4b6222c61bce67340; no voters: 6b7ebd5a66a24240b96752e63ff38268, c16a40ef4eea44bf9ee93bded181a256
21:04:08.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.490134 30500 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:04:08.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.490921 29709 leader_election.cc:336] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391): Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:08.491 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.491071 29709 leader_election.cc:304] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6b7ebd5a66a24240b96752e63ff38268; no voters: 21371941629d4fd4b6222c61bce67340, c16a40ef4eea44bf9ee93bded181a256
21:04:08.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.491370 30501 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:04:08.504 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.545 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.546 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.546 [INFO - pool-38-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.547 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.605 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.606 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.606 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.606 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.607 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] sending RPC to server master-127.28.252.60:39391, [4ms] received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [4ms] delaying RPC due to: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [21ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [21ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [41ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [41ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [62ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [65ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [66ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [81ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [81ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [121ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [122ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [181ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [181ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, deferred=Deferred@1542895887(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@6a0108d1 -> (continuation of Deferred@1177459056 after retry RPC after error@2034529733) -> (continuation of Deferred@395613475 after retry RPC after error@1966166144) -> (continuation of Deferred@1197912655 after retry RPC after error@492178747) -> (continuation of Deferred@785952319 after retry RPC after error@1360154164) -> (continuation of Deferred@596933267 after retry RPC after error@243700751), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@65c46ab6 -> (continuation of Deferred@1177459056 after retry RPC after error@2034529733) -> (continuation of Deferred@395613475 after retry RPC after error@1966166144) -> (continuation of Deferred@1197912655 after retry RPC after error@492178747) -> (continuation of Deferred@785952319 after retry RPC after error@1360154164) -> (continuation of Deferred@596933267 after retry RPC after error@243700751)))
21:04:08.607 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] sending RPC to server master-127.28.252.60:39391, [4ms] received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [4ms] delaying RPC due to: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [21ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [21ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [41ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [41ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [62ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [65ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [66ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [81ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [81ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [121ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [122ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [181ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [181ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, deferred=Deferred@1542895887(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@6a0108d1 -> (continuation of Deferred@1177459056 after retry RPC after error@2034529733) -> (continuation of Deferred@395613475 after retry RPC after error@1966166144) -> (continuation of Deferred@1197912655 after retry RPC after error@492178747) -> (continuation of Deferred@785952319 after retry RPC after error@1360154164) -> (continuation of Deferred@596933267 after retry RPC after error@243700751), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@65c46ab6 -> (continuation of Deferred@1177459056 after retry RPC after error@2034529733) -> (continuation of Deferred@395613475 after retry RPC after error@1966166144) -> (continuation of Deferred@1197912655 after retry RPC after error@492178747) -> (continuation of Deferred@785952319 after retry RPC after error@1360154164) -> (continuation of Deferred@596933267 after retry RPC after error@243700751)))
21:04:08.625 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.626 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.626 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.645 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.646 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.646 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.646 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.647 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.665 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.666 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.666 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.666 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.667 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.685 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.686 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.686 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.686 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.687 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.705 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.706 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.706 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.707 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.725 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.726 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.784 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.787 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.787 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.788 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.805 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.806 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.807 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [2ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [2ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [22ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [22ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [42ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [42ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [62ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [62ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [82ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [82ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [102ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [102ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [162ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [163ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [182ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [182ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, deferred=Deferred@1498823061(state=PENDING, result=null, callback=(continuation of Deferred@2112800438 after retry RPC after error@1467263974) -> (continuation of Deferred@1877428574 after retry RPC after error@127229862) -> (continuation of Deferred@205910676 after retry RPC after error@2115781473) -> (continuation of Deferred@727996485 after retry RPC after error@111233760) -> (continuation of Deferred@626248178 after retry RPC after error@969441650) -> (continuation of Deferred@820461694 after retry RPC after error@1008597704) -> (continuation of Deferred@129677536 after retry RPC after error@2055502452), errback=(continuation of Deferred@2112800438 after retry RPC after error@1467263974) -> (continuation of Deferred@1877428574 after retry RPC after error@127229862) -> (continuation of Deferred@205910676 after retry RPC after error@2115781473) -> (continuation of Deferred@727996485 after retry RPC after error@111233760) -> (continuation of Deferred@626248178 after retry RPC after error@969441650) -> (continuation of Deferred@820461694 after retry RPC after error@1008597704) -> (continuation of Deferred@129677536 after retry RPC after error@2055502452)))
21:04:08.807 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [2ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [2ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [22ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [22ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [42ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [42ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [62ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [62ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [82ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [82ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [102ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [102ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [162ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [163ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [182ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [182ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, deferred=Deferred@1498823061(state=PENDING, result=null, callback=(continuation of Deferred@2112800438 after retry RPC after error@1467263974) -> (continuation of Deferred@1877428574 after retry RPC after error@127229862) -> (continuation of Deferred@205910676 after retry RPC after error@2115781473) -> (continuation of Deferred@727996485 after retry RPC after error@111233760) -> (continuation of Deferred@626248178 after retry RPC after error@969441650) -> (continuation of Deferred@820461694 after retry RPC after error@1008597704) -> (continuation of Deferred@129677536 after retry RPC after error@2055502452), errback=(continuation of Deferred@2112800438 after retry RPC after error@1467263974) -> (continuation of Deferred@1877428574 after retry RPC after error@127229862) -> (continuation of Deferred@205910676 after retry RPC after error@2115781473) -> (continuation of Deferred@727996485 after retry RPC after error@111233760) -> (continuation of Deferred@626248178 after retry RPC after error@969441650) -> (continuation of Deferred@820461694 after retry RPC after error@1008597704) -> (continuation of Deferred@129677536 after retry RPC after error@2055502452)))
21:04:08.825 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.826 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.826 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.826 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.826 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.845 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.846 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.847 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.865 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.867 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.867 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.867 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.867 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.882645 30501 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.882899 30501 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.883608 30501 leader_election.cc:290] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 21371941629d4fd4b6222c61bce67340 (127.28.252.61:36963), c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391)
21:04:08.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.884192 29811 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6b7ebd5a66a24240b96752e63ff38268" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "21371941629d4fd4b6222c61bce67340" is_pre_election: true
21:04:08.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.884449 29811 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b7ebd5a66a24240b96752e63ff38268 in term 2.
21:04:08.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.884975 29709 leader_election.cc:304] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 21371941629d4fd4b6222c61bce67340, 6b7ebd5a66a24240b96752e63ff38268; no voters:
21:04:08.885 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.885303 29709 leader_election.cc:336] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391): Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:08.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.885336 30501 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 2 FOLLOWER]: Leader pre-election won for term 3
21:04:08.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.885445 30501 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:08.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.885556 30501 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 2 FOLLOWER]: Advancing to term 3
21:04:08.886 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.886 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.886 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.887974 30501 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:08.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.888705 30501 leader_election.cc:290] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 3 election: Requested vote from peers 21371941629d4fd4b6222c61bce67340 (127.28.252.61:36963), c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391)
21:04:08.889 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.889433 29811 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6b7ebd5a66a24240b96752e63ff38268" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "21371941629d4fd4b6222c61bce67340"
21:04:08.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.889665 29811 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 2 FOLLOWER]: Advancing to term 3
21:04:08.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.890609 29709 leader_election.cc:336] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391): Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.891947 29811 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b7ebd5a66a24240b96752e63ff38268 in term 3.
21:04:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.892328 29709 leader_election.cc:304] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 21371941629d4fd4b6222c61bce67340, 6b7ebd5a66a24240b96752e63ff38268; no voters: c16a40ef4eea44bf9ee93bded181a256
21:04:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.892583 30501 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 3 FOLLOWER]: Leader election won for term 3
21:04:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.892958 30501 raft_consensus.cc:697] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [term 3 LEADER]: Becoming Leader. State: Replica: 6b7ebd5a66a24240b96752e63ff38268, State: Running, Role: LEADER
21:04:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.893374 30501 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } }
21:04:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.895028 30506 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6b7ebd5a66a24240b96752e63ff38268. Latest consensus state: current_term: 3 leader_uuid: "6b7ebd5a66a24240b96752e63ff38268" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.895378 30506 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [sys.catalog]: This master's current role is: LEADER
21:04:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.896027 30508 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:04:08.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.898312 30508 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback [id=6948525bb3a649e7bdefe5fbd06341e6]
21:04:08.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.898811 30508 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=b558bf9e18f545d29356b296b36e702a]
21:04:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.900442 30508 tablet_loader.cc:96] loaded metadata for tablet 1fea82a6305d458293803989630fc268 (table txn_manager_fallback [id=6948525bb3a649e7bdefe5fbd06341e6])
21:04:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.900799 30508 tablet_loader.cc:96] loaded metadata for tablet 43106605761a4cd28b24398f134279fb (table kudu_system.kudu_transactions [id=b558bf9e18f545d29356b296b36e702a])
21:04:08.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.901083 30508 tablet_loader.cc:96] loaded metadata for tablet 90aac9cc04484a25a2a54dc2674c5dee (table txn_manager_fallback [id=6948525bb3a649e7bdefe5fbd06341e6])
21:04:08.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.901315 30508 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:04:08.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.902110 30508 catalog_manager.cc:1269] Loaded cluster ID: 51f0d1f107744328bb1feb997b8b55af
21:04:08.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.902190 30508 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:04:08.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.903308 30508 catalog_manager.cc:1514] Loading token signing keys...
21:04:08.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.904075 30508 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268: Loaded TSK: 0
21:04:08.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.904881 30508 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:04:08.905 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.906 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.907 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391; will retry
21:04:08.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.914664 30014 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.006s from last keepalive heartbeat (effective timeout is 1.000s)
21:04:08.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.916203 30044 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.28.252.60:39391)
21:04:08.925 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.28.252.60:39391(127.28.252.60:39391)] connecting to peer
21:04:08.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.60:39391(127.28.252.60:39391)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.252.60:39391: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391
21:04:08.927 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:04:08.927 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=6, TimeoutTracker(timeout=250, elapsed=102), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [2ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [2ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [22ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [23ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [42ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [43ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [62ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [62ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [65ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [65ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [82ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [82ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.61:36963: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.28.252.61:36963,127.28.252.62:33507,127.28.252.60:39391) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.61:36963, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.62:33507, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.252.60:39391, [102ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.60:39391: Network error: Failed to connect to peer master-127.28.252.60:39391(127.28.252.60:39391): Connection refused: /127.28.252.60:39391, [102ms] Sub RPC ConnectToMaster: received response from server master-127.28.252.62:33507: OK, deferred=Deferred@382371458(state=PENDING, result=null, callback=(continuation of Deferred@1581891459 after retry RPC after error@1751125826) -> (continuation of Deferred@1584515960 after retry RPC after error@474855586) -> (continuation of Deferred@258713871 after retry RPC after error@1738128831) -> (continuation of Deferred@937660479 after retry RPC after error@1176155849) -> (continuation of Deferred@350964948 after retry RPC after error@1043689395), errback=(continuation of Deferred@1581891459 after retry RPC after error@1751125826) -> (continuation of Deferred@1584515960 after retry RPC after error@474855586) -> (continuation of Deferred@258713871 after retry RPC after error@1738128831) -> (continuation of Deferred@937660479 after retry RPC after error@1176155849) -> (continuation of Deferred@350964948 after retry RPC after error@1043689395))) after lookup
21:04:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.930367 30511 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.967518 29721 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46286:
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:04:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.969637 29721 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.976176 29709 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111) [suppressed 12 similar messages]
21:04:08.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:08.977247 29709 consensus_peers.cc:597] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 -> Peer c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391): Couldn't send request to peer c16a40ef4eea44bf9ee93bded181a256. Status: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:08.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.992060 29811 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [term 3 FOLLOWER]: Refusing update from remote peer 6b7ebd5a66a24240b96752e63ff38268: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
21:04:08.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.994589 30506 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
21:04:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.999174 30534 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6b7ebd5a66a24240b96752e63ff38268. Latest consensus state: current_term: 3 leader_uuid: "6b7ebd5a66a24240b96752e63ff38268" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:08.999817 30534 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [sys.catalog]: This master's current role is: FOLLOWER
21:04:09.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:09.001173 30501 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "6b7ebd5a66a24240b96752e63ff38268" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:09.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:09.001435 30501 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 [sys.catalog]: This master's current role is: LEADER
21:04:09.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:09.003675 30274 heartbeater.cc:646] Failed to heartbeat to 127.28.252.60:39391 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:09.004 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 63bf962d57384f75aea7b1e40362cd23 (127.28.252.1:34617): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1765400647 last_transition_timestamp: 1765400648
21:04:09.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:09.005540 30132 heartbeater.cc:646] Failed to heartbeat to 127.28.252.60:39391 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111)
21:04:09.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:09.005722 30534 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "6b7ebd5a66a24240b96752e63ff38268" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b7ebd5a66a24240b96752e63ff38268" member_type: VOTER last_known_addr { host: "127.28.252.62" port: 33507 } } peers { permanent_uuid: "21371941629d4fd4b6222c61bce67340" member_type: VOTER last_known_addr { host: "127.28.252.61" port: 36963 } } peers { permanent_uuid: "c16a40ef4eea44bf9ee93bded181a256" member_type: VOTER last_known_addr { host: "127.28.252.60" port: 39391 } } }
21:04:09.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:09.005913 30534 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21371941629d4fd4b6222c61bce67340 [sys.catalog]: This master's current role is: FOLLOWER
21:04:09.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:09.485625 29709 consensus_peers.cc:597] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 -> Peer c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391): Couldn't send request to peer c16a40ef4eea44bf9ee93bded181a256. Status: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:04:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:09.984186 29709 consensus_peers.cc:597] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 -> Peer c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391): Couldn't send request to peer c16a40ef4eea44bf9ee93bded181a256. Status: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
21:04:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.002430 30275 heartbeater.cc:499] Master 127.28.252.62:33507 was elected leader, sending a full tablet report...
21:04:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.002447 30133 heartbeater.cc:499] Master 127.28.252.62:33507 was elected leader, sending a full tablet report...
21:04:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.147176 29691 master_runner.cc:186] Error getting master registration for 127.28.252.60:39391: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111),
21:04:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.147392 29691 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:10.206 [DEBUG - main] (Connection.java:575) [peer 2adffa9b6b6d48f08319c66402af52e6(127.28.252.3:36921)] disconnecting while in state READY
21:04:10.206 [DEBUG - main] (Connection.java:575) [peer master-127.28.252.60:39391(127.28.252.60:39391)] disconnecting while in state TERMINATED
21:04:10.206 [DEBUG - main] (Connection.java:575) [peer master-127.28.252.62:33507(127.28.252.62:33507)] disconnecting while in state READY
21:04:10.206 [DEBUG - main] (Connection.java:575) [peer master-127.28.252.61:36963(127.28.252.61:36963)] disconnecting while in state READY
21:04:10.207 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.252.61:36963(127.28.252.61:36963)] handling channelInactive
21:04:10.207 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.252.61:36963(127.28.252.61:36963)] cleaning up while in state READY due to: connection closed
21:04:10.207 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:04:10.208 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 2adffa9b6b6d48f08319c66402af52e6(127.28.252.3:36921)] handling channelInactive
21:04:10.208 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2adffa9b6b6d48f08319c66402af52e6(127.28.252.3:36921)] cleaning up while in state READY due to: connection closed
21:04:10.208 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.252.62:33507(127.28.252.62:33507)] handling channelInactive
21:04:10.208 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.252.62:33507(127.28.252.62:33507)] cleaning up while in state READY due to: connection closed
21:04:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.208721 29680 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu pid 29993
21:04:10.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.221292 30127 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:10.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.321043 30416 heartbeater.cc:499] Master 127.28.252.62:33507 was elected leader, sending a full tablet report...
21:04:10.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.355758 29680 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu with pid 29993
21:04:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.379552 29680 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu pid 30137
21:04:10.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.386300 30269 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.486640 29680 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu with pid 30137
21:04:10.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:10.491595 29709 consensus_peers.cc:597] T 00000000000000000000000000000000 P 6b7ebd5a66a24240b96752e63ff38268 -> Peer c16a40ef4eea44bf9ee93bded181a256 (127.28.252.60:39391): Couldn't send request to peer c16a40ef4eea44bf9ee93bded181a256. Status: Network error: Client connection negotiation failed: client connection to 127.28.252.60:39391: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
21:04:10.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.504801 29680 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu pid 30279
21:04:10.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.511966 30410 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 21:04:10.617359 30297 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.252.2:34483: connect: Connection refused (error 111)
21:04:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.617381 29680 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu with pid 30279
21:04:10.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.635974 29680 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu pid 29691
21:04:10.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.637171 29762 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.756556 29680 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu with pid 29691
21:04:10.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.775321 29680 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu pid 29769
21:04:10.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.776491 29843 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:10.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 21:04:10.887863 29680 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskfRBwJn/build/asan/bin/kudu with pid 29769
21:04:10.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-10T21:04:10Z chronyd exiting
21:04:11.048 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 63bf962d57384f75aea7b1e40362cd23 (127.28.252.1:34617): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1765400647 last_transition_timestamp: 1765400648
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 63bf962d57384f75aea7b1e40362cd23 (127.28.252.1:34617): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1765400647 last_transition_timestamp: 1765400648
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more