21:00:21.162 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:00:21.162 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskV3TXV2/test-tmp
21:00:21.162 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
21:00:21.163 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskV3TXV2/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
21:00:21.164 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:00:21.511 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:00:21.511 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:00:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.515684 28308 mini_oidc.cc:150] Starting JWKS server
21:00:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.518573 28308 webserver.cc:492] Webserver started at http://127.0.0.1:38167/ using document root <none> and password file <none>
21:00:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.518894 28308 mini_oidc.cc:186] Starting OIDC Discovery server
21:00:21.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.519449 28308 webserver.cc:492] Webserver started at http://127.0.0.1:37675/ using document root <none> and password file <none>
21:00:21.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-09T21:00:21Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:00:21.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-09T21:00:21Z Disabled control of system clock
21:00:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.554395 28308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/wal
21:00:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/data
21:00:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/logs
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/data/info.pb
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.62:41881
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.62
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:00:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:00:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.924602 28319 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:00:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.924957 28319 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.925042 28319 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:00:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.933743 28319 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:00:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.933871 28319 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:00:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.933920 28319 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:00:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.933967 28319 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:00:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.934020 28319 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:00:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.934062 28319 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:00:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.934104 28319 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38167/jwks.json
21:00:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.944571 28319 master_runner.cc:386] Master server non-default flags:
21:00:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/data
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/wal
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.62:41881
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/data/info.pb
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.62
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/logs
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 09 Dec 2025 20:43:20 UTC on 24a791456cd2
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9436
21:00:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.946313 28319 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:21.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.948246 28319 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:21.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.956972 28324 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:21.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.956871 28325 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:21.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:21.957932 28327 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:21.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.958691 28319 server_base.cc:1047] running on GCE node
21:00:21.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.959666 28319 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:21.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.961385 28319 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.962651 28319 hybrid_clock.cc:648] HybridClock initialized: now 1765314021962576 us; error 60 us; skew 500 ppm
21:00:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.963078 28319 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.965811 28319 webserver.cc:492] Webserver started at http://127.27.165.62:34355/ using document root <none> and password file <none>
21:00:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.966609 28319 fs_manager.cc:362] Metadata directory not provided
21:00:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.966776 28319 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.967154 28319 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:00:21.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.969774 28319 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/data/instance:
21:00:21.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "466af8d834514b7b9ee9f24cdf09b088"
21:00:21.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:21 on dist-test-slave-4t3w"
21:00:21.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.970647 28319 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/wal/instance:
21:00:21.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "466af8d834514b7b9ee9f24cdf09b088"
21:00:21.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:21 on dist-test-slave-4t3w"
21:00:21.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.977849 28319 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
21:00:21.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.983747 28333 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:00:21.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.985698 28319 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.985947 28319 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/data,/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/wal
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "466af8d834514b7b9ee9f24cdf09b088"
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:21 on dist-test-slave-4t3w"
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:21.986202 28319 fs_report.cc:389] FS layout report
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/wal
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/wal
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/data/data
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:22.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.002336 28319 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:00:22.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.003396 28319 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:22.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.003700 28319 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.030148 28319 rpc_server.cc:307] RPC server started. Bound to: 127.27.165.62:41881
21:00:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.030190 28394 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.165.62:41881 every 8 connection(s)
21:00:22.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.032027 28319 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/data/info.pb
21:00:22.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.036978 28395 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:22.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.041330 28308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu as pid 28319
21:00:22.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.041617 28308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-0/wal/instance
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.044100 28308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/wal
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/data
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/logs
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/data/info.pb
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.61:46277
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.61
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:00:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.043665 28395 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } has no permanent_uuid. Determining permanent_uuid...
21:00:22.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.064414 28395 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } has no permanent_uuid. Determining permanent_uuid...
21:00:22.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.065750 28337 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.165.61:46277: connect: Connection refused (error 111)
21:00:22.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.067721 28395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.61:46277: Network error: Client connection negotiation failed: client connection to 127.27.165.61:46277: connect: Connection refused (error 111)
21:00:22.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.117184 28395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } attempt: 1
21:00:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.119454 28395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.61:46277: Network error: Client connection negotiation failed: client connection to 127.27.165.61:46277: connect: Connection refused (error 111)
21:00:22.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.187741 28395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } attempt: 2
21:00:22.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.189942 28395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.61:46277: Network error: Client connection negotiation failed: client connection to 127.27.165.61:46277: connect: Connection refused (error 111)
21:00:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.281245 28395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } attempt: 3
21:00:22.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.283631 28395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.61:46277: Network error: Client connection negotiation failed: client connection to 127.27.165.61:46277: connect: Connection refused (error 111)
21:00:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.405284 28397 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:00:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.405681 28397 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.405805 28397 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:00:22.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.414458 28397 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:00:22.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.414593 28397 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:00:22.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.414678 28397 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:00:22.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.414764 28397 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:00:22.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.414842 28397 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:00:22.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.414907 28397 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:00:22.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.414984 28397 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38167/jwks.json
21:00:22.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.425601 28397 master_runner.cc:386] Master server non-default flags:
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/data
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/wal
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.61:46277
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/data/info.pb
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.61
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/logs
21:00:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:00:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:00:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:00:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 09 Dec 2025 20:43:20 UTC on 24a791456cd2
21:00:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9436
21:00:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.426926 28395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } attempt: 4
21:00:22.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.427964 28397 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.429085 28395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.61:46277: Network error: Client connection negotiation failed: client connection to 127.27.165.61:46277: connect: Connection refused (error 111)
21:00:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.430197 28397 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.439105 28405 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.439098 28406 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.439946 28408 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.440240 28397 server_base.cc:1047] running on GCE node
21:00:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.441143 28397 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.442896 28397 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:22.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.444211 28397 hybrid_clock.cc:648] HybridClock initialized: now 1765314022444115 us; error 70 us; skew 500 ppm
21:00:22.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.444645 28397 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:22.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.447757 28397 webserver.cc:492] Webserver started at http://127.27.165.61:34245/ using document root <none> and password file <none>
21:00:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.448678 28397 fs_manager.cc:362] Metadata directory not provided
21:00:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.448840 28397 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:22.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.449221 28397 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:00:22.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.452064 28397 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/data/instance:
21:00:22.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da910329c4c84f5f8faa465510be0b11"
21:00:22.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:22 on dist-test-slave-4t3w"
21:00:22.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.453055 28397 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/wal/instance:
21:00:22.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da910329c4c84f5f8faa465510be0b11"
21:00:22.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:22 on dist-test-slave-4t3w"
21:00:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.459318 28397 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
21:00:22.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.463258 28414 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:00:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.464850 28397 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
21:00:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.465076 28397 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/data,/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/wal
21:00:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da910329c4c84f5f8faa465510be0b11"
21:00:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:22 on dist-test-slave-4t3w"
21:00:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.465314 28397 fs_report.cc:389] FS layout report
21:00:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/wal
21:00:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/wal
21:00:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/data/data
21:00:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:22.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.481668 28397 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:00:22.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.482815 28397 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:22.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.483186 28397 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:22.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.506847 28397 rpc_server.cc:307] RPC server started. Bound to: 127.27.165.61:46277
21:00:22.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.506896 28475 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.165.61:46277 every 8 connection(s)
21:00:22.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.508489 28397 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/data/info.pb
21:00:22.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.513078 28476 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:22.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.516571 28308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu as pid 28397
21:00:22.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.516753 28308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-1/wal/instance
21:00:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.517756 28308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/wal
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/data
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/logs
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/data/info.pb
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.60:39093
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.60
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json
21:00:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:00:22.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:00:22.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.519275 28476 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } has no permanent_uuid. Determining permanent_uuid...
21:00:22.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.534267 28476 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } has no permanent_uuid. Determining permanent_uuid...
21:00:22.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.544256 28476 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } has no permanent_uuid. Determining permanent_uuid...
21:00:22.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.545447 28417 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:22.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.547174 28476 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.60:39093: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:22.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.596513 28476 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } attempt: 1
21:00:22.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.598610 28476 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.60:39093: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:22.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.666939 28476 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } attempt: 2
21:00:22.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.669165 28476 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.60:39093: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.728363 28395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } attempt: 5
21:00:22.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.735548 28395 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } has no permanent_uuid. Determining permanent_uuid...
21:00:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.737552 28395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.60:39093: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.760437 28476 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } attempt: 3
21:00:22.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.763010 28476 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.60:39093: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:22.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.788797 28395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } attempt: 1
21:00:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.790936 28395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.60:39093: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.859196 28395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } attempt: 2
21:00:22.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.861385 28395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.60:39093: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.877368 28478 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:00:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.877736 28478 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.877820 28478 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:00:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.886456 28478 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:00:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.886576 28478 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:00:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.886626 28478 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:00:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.886667 28478 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:00:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.886746 28478 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:00:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.886791 28478 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:00:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.886835 28478 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38167/jwks.json
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.897298 28478 master_runner.cc:386] Master server non-default flags:
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/data
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/wal
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.60:39093
21:00:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38167/jwks.json
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/data/info.pb
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.60
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/logs
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 09 Dec 2025 20:43:20 UTC on 24a791456cd2
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9436
21:00:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:22.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.899056 28478 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:22.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.901022 28478 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.906291 28476 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } attempt: 4
21:00:22.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.909349 28476 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.60:39093: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:22.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.910652 28486 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:22.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.911757 28487 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:22.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.912650 28489 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:22.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.913506 28478 server_base.cc:1047] running on GCE node
21:00:22.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.914393 28478 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:22.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.916059 28478 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:22.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.917318 28478 hybrid_clock.cc:648] HybridClock initialized: now 1765314022917253 us; error 40 us; skew 500 ppm
21:00:22.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.917706 28478 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:22.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.920387 28478 webserver.cc:492] Webserver started at http://127.27.165.60:45623/ using document root <none> and password file <none>
21:00:22.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.921247 28478 fs_manager.cc:362] Metadata directory not provided
21:00:22.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.921403 28478 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:22.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.921777 28478 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:00:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.924499 28478 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/data/instance:
21:00:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "151e9e3eb2c046a4b4715f63e74ec358"
21:00:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:22 on dist-test-slave-4t3w"
21:00:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.925351 28478 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/wal/instance:
21:00:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "151e9e3eb2c046a4b4715f63e74ec358"
21:00:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:22 on dist-test-slave-4t3w"
21:00:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.931519 28478 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.003s sys 0.005s
21:00:22.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.935654 28495 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:00:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.937198 28478 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:00:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.937451 28478 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/data,/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/wal
21:00:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "151e9e3eb2c046a4b4715f63e74ec358"
21:00:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:22 on dist-test-slave-4t3w"
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.937700 28478 fs_report.cc:389] FS layout report
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/wal
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/wal
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/data/data
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.956837 28478 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:00:22.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.958117 28478 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:22.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.958490 28478 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.967622 28395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } attempt: 3
21:00:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:22.970276 28395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.165.60:39093: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.985664 28478 rpc_server.cc:307] RPC server started. Bound to: 127.27.165.60:39093
21:00:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.985683 28556 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.165.60:39093 every 8 connection(s)
21:00:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.987504 28478 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/data/info.pb
21:00:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.988670 28308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu as pid 28478
21:00:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.988894 28308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/master-2/wal/instance
21:00:22.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.992640 28557 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:22.999843 28557 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } has no permanent_uuid. Determining permanent_uuid...
21:00:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.015889 28557 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } has no permanent_uuid. Determining permanent_uuid...
21:00:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.024096 28557 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } has no permanent_uuid. Determining permanent_uuid...
21:00:23.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.034785 28557 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358: Bootstrap starting.
21:00:23.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.038669 28557 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358: Neither blocks nor log segments found. Creating new log.
21:00:23.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.039927 28557 log.cc:826] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358: Log is configured to *not* fsync() on all Append() calls
21:00:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.043206 28557 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358: No bootstrap required, opened a new log
21:00:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.049232 28557 raft_consensus.cc:359] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.049640 28557 raft_consensus.cc:385] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.049762 28557 raft_consensus.cc:740] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 151e9e3eb2c046a4b4715f63e74ec358, State: Initialized, Role: FOLLOWER
21:00:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.050506 28557 consensus_queue.cc:260] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:23.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.051647 28564 sys_catalog.cc:455] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:23.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.051983 28564 sys_catalog.cc:458] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [sys.catalog]: This master's current role is: FOLLOWER
21:00:23.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.053215 28557 sys_catalog.cc:565] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [sys.catalog]: configured and running, proceeding with master startup.
21:00:23.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.065284 28575 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:00:23.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.065447 28575 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:00:23.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.062163 28478 master_runner.cc:186] Error getting master registration for 127.27.165.62:41881: OK, instance_id { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" instance_seqno: 1765314022023928 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:00:23.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.067602 28478 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:00:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.121737 28564 raft_consensus.cc:493] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:00:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.122030 28564 raft_consensus.cc:515] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.123760 28564 leader_election.cc:290] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), da910329c4c84f5f8faa465510be0b11 (127.27.165.61:46277)
21:00:23.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.147527 28395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } attempt: 4
21:00:23.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.161188 28395 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088: Bootstrap starting.
21:00:23.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.166100 28395 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088: Neither blocks nor log segments found. Creating new log.
21:00:23.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.167764 28395 log.cc:826] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088: Log is configured to *not* fsync() on all Append() calls
21:00:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.171414 28395 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088: No bootstrap required, opened a new log
21:00:23.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.179987 28395 raft_consensus.cc:359] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:23.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.180413 28395 raft_consensus.cc:385] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:23.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.180552 28395 raft_consensus.cc:740] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 466af8d834514b7b9ee9f24cdf09b088, State: Initialized, Role: FOLLOWER
21:00:23.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.181368 28395 consensus_queue.cc:260] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:23.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.182648 28578 sys_catalog.cc:455] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:23.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.183465 28578 sys_catalog.cc:458] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [sys.catalog]: This master's current role is: FOLLOWER
21:00:23.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.184213 28395 sys_catalog.cc:565] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [sys.catalog]: configured and running, proceeding with master startup.
21:00:23.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.184244 28360 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "151e9e3eb2c046a4b4715f63e74ec358" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466af8d834514b7b9ee9f24cdf09b088" is_pre_election: true
21:00:23.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.185051 28360 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 151e9e3eb2c046a4b4715f63e74ec358 in term 0.
21:00:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.186034 28498 leader_election.cc:304] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088; no voters:
21:00:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.186612 28564 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:00:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.186810 28564 raft_consensus.cc:493] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:00:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.186925 28564 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [term 0 FOLLOWER]: Advancing to term 1
21:00:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.190207 28564 raft_consensus.cc:515] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:23.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.194514 28360 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "151e9e3eb2c046a4b4715f63e74ec358" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "466af8d834514b7b9ee9f24cdf09b088"
21:00:23.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.194795 28360 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 0 FOLLOWER]: Advancing to term 1
21:00:23.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.198184 28360 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 151e9e3eb2c046a4b4715f63e74ec358 in term 1.
21:00:23.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.198586 28564 leader_election.cc:290] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [CANDIDATE]: Term 1 election: Requested vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), da910329c4c84f5f8faa465510be0b11 (127.27.165.61:46277)
21:00:23.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.199075 28498 leader_election.cc:304] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088; no voters:
21:00:23.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.199484 28564 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [term 1 FOLLOWER]: Leader election won for term 1
21:00:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.200678 28564 raft_consensus.cc:697] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [term 1 LEADER]: Becoming Leader. State: Replica: 151e9e3eb2c046a4b4715f63e74ec358, State: Running, Role: LEADER
21:00:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.201412 28564 consensus_queue.cc:237] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:23.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.205256 28583 sys_catalog.cc:455] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 151e9e3eb2c046a4b4715f63e74ec358. Latest consensus state: current_term: 1 leader_uuid: "151e9e3eb2c046a4b4715f63e74ec358" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.206002 28583 sys_catalog.cc:458] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [sys.catalog]: This master's current role is: LEADER
21:00:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.207764 28592 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:00:23.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.208603 28476 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } attempt: 5
21:00:23.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.211208 28501 tablet.cc:2378] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:00:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.216936 28592 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:00:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.229938 28591 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:00:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.230355 28591 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:00:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.233505 28476 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11: Bootstrap starting.
21:00:23.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.243891 28360 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 1 FOLLOWER]: Refusing update from remote peer 151e9e3eb2c046a4b4715f63e74ec358: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:00:23.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.219542 28319 master_runner.cc:186] Error getting master registration for 127.27.165.61:46277: OK, instance_id { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" instance_seqno: 1765314022501016 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:00:23.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.244406 28319 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:00:23.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.245361 28476 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11: Neither blocks nor log segments found. Creating new log.
21:00:23.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.246148 28583 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [LEADER]: Connected to new peer: Peer: permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:00:23.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.247308 28476 log.cc:826] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11: Log is configured to *not* fsync() on all Append() calls
21:00:23.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.256857 28339 tablet.cc:2378] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:00:23.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.272286 28578 sys_catalog.cc:455] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 151e9e3eb2c046a4b4715f63e74ec358. Latest consensus state: current_term: 1 leader_uuid: "151e9e3eb2c046a4b4715f63e74ec358" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:23.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.272604 28578 sys_catalog.cc:458] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [sys.catalog]: This master's current role is: FOLLOWER
21:00:23.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.273710 28594 mvcc.cc:204] Tried to move back new op lower bound from 7230726239190421504 to 7230726239038730240. Current Snapshot: MvccSnapshot[applied={T|T < 7230726239190421504}]
21:00:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.279343 28596 mvcc.cc:204] Tried to move back new op lower bound from 7230726239190421504 to 7230726239038730240. Current Snapshot: MvccSnapshot[applied={T|T < 7230726239190421504}]
21:00:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.283212 28564 sys_catalog.cc:455] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "151e9e3eb2c046a4b4715f63e74ec358" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.283612 28564 sys_catalog.cc:458] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [sys.catalog]: This master's current role is: LEADER
21:00:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.285043 28578 sys_catalog.cc:455] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "151e9e3eb2c046a4b4715f63e74ec358" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.285315 28578 sys_catalog.cc:458] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [sys.catalog]: This master's current role is: FOLLOWER
21:00:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.291906 28592 catalog_manager.cc:1357] Generated new cluster ID: d31db4baae2b4fe98542ff4d78378ffb
21:00:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.292068 28592 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:00:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.297973 28476 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11: No bootstrap required, opened a new log
21:00:23.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.317835 28476 raft_consensus.cc:359] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:23.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.322338 28476 raft_consensus.cc:385] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:23.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.322527 28476 raft_consensus.cc:740] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: da910329c4c84f5f8faa465510be0b11, State: Initialized, Role: FOLLOWER
21:00:23.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.323336 28592 catalog_manager.cc:1380] Generated new certificate authority record
21:00:23.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.323427 28476 consensus_queue.cc:260] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:23.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.328130 28592 catalog_manager.cc:1514] Loading token signing keys...
21:00:23.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.330755 28600 sys_catalog.cc:455] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:23.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.331168 28600 sys_catalog.cc:458] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [sys.catalog]: This master's current role is: FOLLOWER
21:00:23.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.333325 28476 sys_catalog.cc:565] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [sys.catalog]: configured and running, proceeding with master startup.
21:00:23.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.333374 28441 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "151e9e3eb2c046a4b4715f63e74ec358" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "da910329c4c84f5f8faa465510be0b11" is_pre_election: true
21:00:23.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.334070 28441 raft_consensus.cc:2468] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 151e9e3eb2c046a4b4715f63e74ec358 in term 0.
21:00:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.335234 28439 raft_consensus.cc:3060] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 0 FOLLOWER]: Advancing to term 1
21:00:23.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.334622 28440 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "151e9e3eb2c046a4b4715f63e74ec358" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "da910329c4c84f5f8faa465510be0b11"
21:00:23.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.341411 28439 raft_consensus.cc:1275] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 1 FOLLOWER]: Refusing update from remote peer 151e9e3eb2c046a4b4715f63e74ec358: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:00:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.342617 28564 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [LEADER]: Connected to new peer: Peer: permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:00:23.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.372710 28592 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358: Generated new TSK 0
21:00:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.373952 28592 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:00:23.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.395535 28600 sys_catalog.cc:455] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 151e9e3eb2c046a4b4715f63e74ec358. Latest consensus state: current_term: 1 leader_uuid: "151e9e3eb2c046a4b4715f63e74ec358" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:23.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.395855 28600 sys_catalog.cc:458] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [sys.catalog]: This master's current role is: FOLLOWER
21:00:23.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.396063 28583 sys_catalog.cc:455] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "151e9e3eb2c046a4b4715f63e74ec358" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:23.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.396330 28583 sys_catalog.cc:458] T 00000000000000000000000000000000 P 151e9e3eb2c046a4b4715f63e74ec358 [sys.catalog]: This master's current role is: LEADER
21:00:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.399219 28608 sys_catalog.cc:455] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "151e9e3eb2c046a4b4715f63e74ec358" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.400139 28608 sys_catalog.cc:458] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [sys.catalog]: This master's current role is: FOLLOWER
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.407256 28308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/wal
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/data
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/logs
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/data/info.pb
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.1:0
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.165.1
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.1
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:00:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:00:23.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.462399 28617 catalog_manager.cc:1269] Loaded cluster ID: d31db4baae2b4fe98542ff4d78378ffb
21:00:23.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.462549 28617 catalog_manager.cc:1562] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11: loading cluster ID for follower catalog manager: success
21:00:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.467029 28617 catalog_manager.cc:1584] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11: acquiring CA information for follower catalog manager: success
21:00:23.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.475736 28617 catalog_manager.cc:1612] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:00:23.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.969661 28615 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:00:23.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.970124 28615 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:23.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.970279 28615 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:00:23.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.979871 28615 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:00:23.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.980082 28615 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:00:23.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.980157 28615 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:00:23.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.980216 28615 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:00:23.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:23.980361 28615 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.165.1
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.994829 28615 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/data
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/wal
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.1:0
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/data/info.pb
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.1
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.165.1
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/logs
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 09 Dec 2025 20:43:20 UTC on 24a791456cd2
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9436
21:00:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:23.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.997047 28615 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:23.999626 28615 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.011665 28624 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.011749 28623 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:24.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.013963 28626 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:24.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.014317 28615 server_base.cc:1047] running on GCE node
21:00:24.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.015018 28615 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:24.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.016044 28615 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:24.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.017303 28615 hybrid_clock.cc:648] HybridClock initialized: now 1765314024017235 us; error 43 us; skew 500 ppm
21:00:24.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.017679 28615 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.020467 28615 webserver.cc:492] Webserver started at http://127.27.165.1:44393/ using document root <none> and password file <none>
21:00:24.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.021344 28615 fs_manager.cc:362] Metadata directory not provided
21:00:24.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.021512 28615 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:24.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.021926 28615 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:00:24.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.024894 28615 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/data/instance:
21:00:24.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22a807342d864d3b97a1a97e92484b22"
21:00:24.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:24 on dist-test-slave-4t3w"
21:00:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.025942 28615 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/wal/instance:
21:00:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22a807342d864d3b97a1a97e92484b22"
21:00:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:24 on dist-test-slave-4t3w"
21:00:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.032992 28615 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.005s
21:00:24.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.038069 28632 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:00:24.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.040239 28615 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
21:00:24.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.040477 28615 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/data,/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/wal
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22a807342d864d3b97a1a97e92484b22"
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:24 on dist-test-slave-4t3w"
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.040745 28615 fs_report.cc:389] FS layout report
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/wal
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/wal
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/data/data
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:24.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.062436 28615 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:00:24.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.063490 28615 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:24.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.063773 28615 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.067190 28615 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:00:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.067365 28615 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:00:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.067492 28615 ts_tablet_manager.cc:616] Registered 0 tablets
21:00:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.067579 28615 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:00:24.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.134531 28615 rpc_server.cc:307] RPC server started. Bound to: 127.27.165.1:45079
21:00:24.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.134598 28756 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.165.1:45079 every 8 connection(s)
21:00:24.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.135954 28615 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/data/info.pb
21:00:24.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.139014 28758 heartbeater.cc:344] Connected to a master server at 127.27.165.62:41881
21:00:24.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.139420 28758 heartbeater.cc:461] Registering TS with master...
21:00:24.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.140602 28758 heartbeater.cc:507] Master 127.27.165.62:41881 requested a full tablet report, sending...
21:00:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.143462 28350 ts_manager.cc:194] Registered new tserver with Master: 22a807342d864d3b97a1a97e92484b22 (127.27.165.1:45079)
21:00:24.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.144711 28308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu as pid 28615
21:00:24.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.144878 28308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-0/wal/instance
21:00:24.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.148329 28759 heartbeater.cc:344] Connected to a master server at 127.27.165.61:46277
21:00:24.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.148501 28759 heartbeater.cc:461] Registering TS with master...
21:00:24.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.149062 28759 heartbeater.cc:507] Master 127.27.165.61:46277 requested a full tablet report, sending...
21:00:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.149802 28757 heartbeater.cc:344] Connected to a master server at 127.27.165.60:39093
21:00:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.149945 28757 heartbeater.cc:461] Registering TS with master...
21:00:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.150509 28757 heartbeater.cc:507] Master 127.27.165.60:39093 requested a full tablet report, sending...
21:00:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.150750 28431 ts_manager.cc:194] Registered new tserver with Master: 22a807342d864d3b97a1a97e92484b22 (127.27.165.1:45079)
21:00:24.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.152604 28308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:24.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:24.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/wal
21:00:24.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/data
21:00:24.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:24.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:24.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:24.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/logs
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/data/info.pb
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.2:0
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.165.2
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.2
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:00:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:00:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:00:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.152630 28512 ts_manager.cc:194] Registered new tserver with Master: 22a807342d864d3b97a1a97e92484b22 (127.27.165.1:45079)
21:00:24.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.154947 28512 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.165.1:49199
21:00:24.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.232951 28591 catalog_manager.cc:1269] Loaded cluster ID: d31db4baae2b4fe98542ff4d78378ffb
21:00:24.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.233088 28591 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088: loading cluster ID for follower catalog manager: success
21:00:24.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.236840 28591 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088: acquiring CA information for follower catalog manager: success
21:00:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.239369 28591 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:00:24.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.518710 28762 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:00:24.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.519106 28762 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:24.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.519232 28762 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:00:24.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.528261 28762 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:00:24.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.528472 28762 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:00:24.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.528554 28762 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:00:24.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.528609 28762 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:00:24.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.528738 28762 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.165.2
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.539973 28762 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/data
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/wal
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.2:0
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/data/info.pb
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.2
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.165.2
21:00:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/logs
21:00:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:00:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:00:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 09 Dec 2025 20:43:20 UTC on 24a791456cd2
21:00:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9436
21:00:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:24.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.542184 28762 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:24.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.544359 28762 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:24.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.555315 28769 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:24.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.555341 28768 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:24.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:24.555773 28771 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:24.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.557164 28762 server_base.cc:1047] running on GCE node
21:00:24.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.557833 28762 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.558965 28762 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:24.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.560186 28762 hybrid_clock.cc:648] HybridClock initialized: now 1765314024560129 us; error 44 us; skew 500 ppm
21:00:24.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.560489 28762 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:24.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.563247 28762 webserver.cc:492] Webserver started at http://127.27.165.2:42497/ using document root <none> and password file <none>
21:00:24.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.564054 28762 fs_manager.cc:362] Metadata directory not provided
21:00:24.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.564184 28762 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:24.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.564505 28762 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:00:24.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.567229 28762 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/data/instance:
21:00:24.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "07a12c4fa3e24352a99078a970f5d452"
21:00:24.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:24 on dist-test-slave-4t3w"
21:00:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.568032 28762 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/wal/instance:
21:00:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "07a12c4fa3e24352a99078a970f5d452"
21:00:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:24 on dist-test-slave-4t3w"
21:00:24.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.573977 28762 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
21:00:24.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.578102 28777 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.579825 28762 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.003s
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.580005 28762 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/data,/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/wal
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "07a12c4fa3e24352a99078a970f5d452"
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:24 on dist-test-slave-4t3w"
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.580231 28762 fs_report.cc:389] FS layout report
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/wal
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/wal
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/data/data
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.603492 28762 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:00:24.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.604425 28762 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:24.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.604746 28762 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:24.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.608135 28762 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:00:24.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.608291 28762 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:00:24.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.608430 28762 ts_tablet_manager.cc:616] Registered 0 tablets
21:00:24.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.608539 28762 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:00:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.668638 28762 rpc_server.cc:307] RPC server started. Bound to: 127.27.165.2:42895
21:00:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.668699 28899 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.165.2:42895 every 8 connection(s)
21:00:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.670094 28762 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/data/info.pb
21:00:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.673133 28308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu as pid 28762
21:00:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.673384 28308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-1/wal/instance
21:00:24.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.673972 28901 heartbeater.cc:344] Connected to a master server at 127.27.165.62:41881
21:00:24.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.674546 28901 heartbeater.cc:461] Registering TS with master...
21:00:24.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.676446 28901 heartbeater.cc:507] Master 127.27.165.62:41881 requested a full tablet report, sending...
21:00:24.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.677822 28308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/wal
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/data
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/logs
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/data/info.pb
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.3:0
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.165.3
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.3
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:00:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:00:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.679764 28350 ts_manager.cc:194] Registered new tserver with Master: 07a12c4fa3e24352a99078a970f5d452 (127.27.165.2:42895)
21:00:24.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.682417 28902 heartbeater.cc:344] Connected to a master server at 127.27.165.61:46277
21:00:24.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.682753 28902 heartbeater.cc:461] Registering TS with master...
21:00:24.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.683526 28902 heartbeater.cc:507] Master 127.27.165.61:46277 requested a full tablet report, sending...
21:00:24.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.683914 28900 heartbeater.cc:344] Connected to a master server at 127.27.165.60:39093
21:00:24.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.684048 28900 heartbeater.cc:461] Registering TS with master...
21:00:24.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.684546 28900 heartbeater.cc:507] Master 127.27.165.60:39093 requested a full tablet report, sending...
21:00:24.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.684800 28431 ts_manager.cc:194] Registered new tserver with Master: 07a12c4fa3e24352a99078a970f5d452 (127.27.165.2:42895)
21:00:24.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.685642 28512 ts_manager.cc:194] Registered new tserver with Master: 07a12c4fa3e24352a99078a970f5d452 (127.27.165.2:42895)
21:00:24.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:24.686923 28512 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.165.2:50307
21:00:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.037498 28905 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:00:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.037850 28905 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:00:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.037951 28905 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:00:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.046976 28905 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:00:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.047129 28905 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:00:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.047179 28905 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:00:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.047216 28905 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:00:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.047298 28905 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.165.3
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.058168 28905 tablet_server_runner.cc:78] Tablet server non-default flags:
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.165.20:44995
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/data
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/wal
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.165.3:0
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/data/info.pb
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.165.3
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.165.3
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/logs
21:00:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:00:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:00:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:00:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:00:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:00:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:00:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:00:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 09 Dec 2025 20:43:20 UTC on 24a791456cd2
21:00:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9436
21:00:25.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:00:25.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.059932 28905 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:00:25.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.061879 28905 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:00:25.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.072090 28910 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:25.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.072305 28911 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:25.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.073218 28913 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:00:25.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.073695 28905 server_base.cc:1047] running on GCE node
21:00:25.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.074369 28905 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:00:25.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.075543 28905 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:00:25.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.076781 28905 hybrid_clock.cc:648] HybridClock initialized: now 1765314025076699 us; error 60 us; skew 500 ppm
21:00:25.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.077088 28905 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:00:25.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.079661 28905 webserver.cc:492] Webserver started at http://127.27.165.3:33213/ using document root <none> and password file <none>
21:00:25.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.080438 28905 fs_manager.cc:362] Metadata directory not provided
21:00:25.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.080556 28905 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:00:25.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.080870 28905 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:00:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.083606 28905 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/data/instance:
21:00:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd3b32698ab94b7f9e9be9c587edfb61"
21:00:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:25 on dist-test-slave-4t3w"
21:00:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.084379 28905 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/wal/instance:
21:00:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd3b32698ab94b7f9e9be9c587edfb61"
21:00:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:25 on dist-test-slave-4t3w"
21:00:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.090454 28905 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.002s sys 0.005s
21:00:25.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.094635 28919 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:00:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.096313 28905 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
21:00:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.096505 28905 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/data,/tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/wal
21:00:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd3b32698ab94b7f9e9be9c587edfb61"
21:00:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-09 21:00:25 on dist-test-slave-4t3w"
21:00:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.096714 28905 fs_report.cc:389] FS layout report
21:00:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:00:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/wal
21:00:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/wal
21:00:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/data/data
21:00:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:00:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:00:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:00:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:00:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:00:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:00:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:00:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:00:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:00:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:00:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:00:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.137097 28905 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:00:25.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.138105 28905 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:00:25.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.138441 28905 kserver.cc:163] Server-wide thread pool size limit: 3276
21:00:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.142017 28905 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:00:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.142192 28905 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:00:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.142328 28905 ts_tablet_manager.cc:616] Registered 0 tablets
21:00:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.142407 28905 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:00:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.158138 28757 heartbeater.cc:499] Master 127.27.165.60:39093 was elected leader, sending a full tablet report...
21:00:25.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.216562 28905 rpc_server.cc:307] RPC server started. Bound to: 127.27.165.3:36663
21:00:25.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.216614 29041 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.165.3:36663 every 8 connection(s)
21:00:25.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.218056 28905 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/data/info.pb
21:00:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.221236 29043 heartbeater.cc:344] Connected to a master server at 127.27.165.62:41881
21:00:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.221978 29043 heartbeater.cc:461] Registering TS with master...
21:00:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.221973 28308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu as pid 28905
21:00:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.222150 28308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV3TXV2/test-tmp/mini-kudu-cluster5894658161344693756/ts-2/wal/instance
21:00:25.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.223544 29043 heartbeater.cc:507] Master 127.27.165.62:41881 requested a full tablet report, sending...
21:00:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.226580 28350 ts_manager.cc:194] Registered new tserver with Master: cd3b32698ab94b7f9e9be9c587edfb61 (127.27.165.3:36663)
21:00:25.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.229602 29042 heartbeater.cc:344] Connected to a master server at 127.27.165.60:39093
21:00:25.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.230094 29042 heartbeater.cc:461] Registering TS with master...
21:00:25.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.230485 29044 heartbeater.cc:344] Connected to a master server at 127.27.165.61:46277
21:00:25.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.230644 29044 heartbeater.cc:461] Registering TS with master...
21:00:25.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.230752 29042 heartbeater.cc:507] Master 127.27.165.60:39093 requested a full tablet report, sending...
21:00:25.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.231244 29044 heartbeater.cc:507] Master 127.27.165.61:46277 requested a full tablet report, sending...
21:00:25.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.231995 28511 ts_manager.cc:194] Registered new tserver with Master: cd3b32698ab94b7f9e9be9c587edfb61 (127.27.165.3:36663)
21:00:25.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.232426 28431 ts_manager.cc:194] Registered new tserver with Master: cd3b32698ab94b7f9e9be9c587edfb61 (127.27.165.3:36663)
21:00:25.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.233209 28511 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.165.3:47339
21:00:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.248646 28308 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:00:25.250 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:00:25.250 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:00:25.252 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.165.62"
port: 41881
}
bound_http_address {
host: "127.27.165.62"
port: 34355
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.165.61"
port: 46277
}
bound_http_address {
host: "127.27.165.61"
port: 34245
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.165.60"
port: 39093
}
bound_http_address {
host: "127.27.165.60"
port: 45623
}
}
}
21:00:25.252 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:00:25.252 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.165.1"
port: 45079
}
bound_http_address {
host: "127.27.165.1"
port: 44393
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.165.2"
port: 42895
}
bound_http_address {
host: "127.27.165.2"
port: 42497
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.165.3"
port: 36663
}
bound_http_address {
host: "127.27.165.3"
port: 33213
}
}
}
21:00:25.253 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:00:25.254 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:00:25.256 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:25.257 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.165.61:46277(127.27.165.61:46277)] connecting to peer
21:00:25.258 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:25.260 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.165.60:39093(127.27.165.60:39093)] Successfully connected to peer
21:00:25.260 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.27.165.61:46277(127.27.165.61:46277)] Successfully connected to peer
21:00:25.261 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.165.62:41881(127.27.165.62:41881)] Successfully connected to peer
21:00:25.262 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.27.165.60
21:00:25.262 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.27.165.61
21:00:25.266 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.27.165.62
21:00:25.277 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x91f2f7c2, L:/127.0.0.1:42414 - R:/127.27.165.61:46277] using SASL/PLAIN
21:00:25.283 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xbc51b3ae, L:/127.0.0.1:54746 - R:/127.27.165.60:39093] using SASL/PLAIN
21:00:25.285 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xec866440, L:/127.0.0.1:50348 - R:/127.27.165.62:41881] using SASL/PLAIN
21:00:25.287 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:00:25.287 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=31), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [24ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [31ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: OK, deferred=null) after lookup
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.289131 28511 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54746:
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:00:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:00:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
21:00:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:00:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
21:00:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
21:00:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.291570 28511 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:00:25.333 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 99e1ceca529848e3ade6b45912eec9e9
21:00:25.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.386837 28835 tablet_service.cc:1505] Processing CreateTablet for tablet fb991d2001c245ae8f513d61a903aafd (DEFAULT_TABLE table=txn_manager_fallback [id=99e1ceca529848e3ade6b45912eec9e9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:00:25.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.386824 28977 tablet_service.cc:1505] Processing CreateTablet for tablet fb991d2001c245ae8f513d61a903aafd (DEFAULT_TABLE table=txn_manager_fallback [id=99e1ceca529848e3ade6b45912eec9e9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:00:25.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.388566 28835 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb991d2001c245ae8f513d61a903aafd. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:25.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.389015 28977 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb991d2001c245ae8f513d61a903aafd. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.390169 28683 tablet_service.cc:1505] Processing CreateTablet for tablet fb991d2001c245ae8f513d61a903aafd (DEFAULT_TABLE table=txn_manager_fallback [id=99e1ceca529848e3ade6b45912eec9e9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:00:25.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.391443 28683 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb991d2001c245ae8f513d61a903aafd. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:25.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.392285 28834 tablet_service.cc:1505] Processing CreateTablet for tablet 5ff37ea97c994540928e18315264109d (DEFAULT_TABLE table=txn_manager_fallback [id=99e1ceca529848e3ade6b45912eec9e9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:00:25.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.392823 28834 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5ff37ea97c994540928e18315264109d. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:25.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.395095 28682 tablet_service.cc:1505] Processing CreateTablet for tablet 5ff37ea97c994540928e18315264109d (DEFAULT_TABLE table=txn_manager_fallback [id=99e1ceca529848e3ade6b45912eec9e9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:00:25.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.395625 28682 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5ff37ea97c994540928e18315264109d. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:25.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.396350 28976 tablet_service.cc:1505] Processing CreateTablet for tablet 5ff37ea97c994540928e18315264109d (DEFAULT_TABLE table=txn_manager_fallback [id=99e1ceca529848e3ade6b45912eec9e9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:00:25.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.396925 28976 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5ff37ea97c994540928e18315264109d. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:25.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.415360 29068 tablet_bootstrap.cc:492] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61: Bootstrap starting.
21:00:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.419194 29070 tablet_bootstrap.cc:492] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22: Bootstrap starting.
21:00:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.420117 29068 tablet_bootstrap.cc:654] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61: Neither blocks nor log segments found. Creating new log.
21:00:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.420586 29069 tablet_bootstrap.cc:492] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452: Bootstrap starting.
21:00:25.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.421589 29068 log.cc:826] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61: Log is configured to *not* fsync() on all Append() calls
21:00:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.424012 29070 tablet_bootstrap.cc:654] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22: Neither blocks nor log segments found. Creating new log.
21:00:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.424300 29069 tablet_bootstrap.cc:654] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452: Neither blocks nor log segments found. Creating new log.
21:00:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.425096 29068 tablet_bootstrap.cc:492] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61: No bootstrap required, opened a new log
21:00:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.425413 29068 ts_tablet_manager.cc:1403] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.004s
21:00:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.425629 29070 log.cc:826] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22: Log is configured to *not* fsync() on all Append() calls
21:00:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.425688 29069 log.cc:826] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452: Log is configured to *not* fsync() on all Append() calls
21:00:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.428629 29070 tablet_bootstrap.cc:492] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22: No bootstrap required, opened a new log
21:00:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.428849 29069 tablet_bootstrap.cc:492] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452: No bootstrap required, opened a new log
21:00:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.428956 29070 ts_tablet_manager.cc:1403] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.004s
21:00:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.429149 29069 ts_tablet_manager.cc:1403] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452: Time spent bootstrapping tablet: real 0.009s user 0.006s sys 0.000s
21:00:25.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.433252 29068 raft_consensus.cc:359] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } }
21:00:25.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.433769 29068 raft_consensus.cc:385] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.433880 29068 raft_consensus.cc:740] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd3b32698ab94b7f9e9be9c587edfb61, State: Initialized, Role: FOLLOWER
21:00:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.434343 29070 raft_consensus.cc:359] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.434687 29070 raft_consensus.cc:385] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.434831 29070 raft_consensus.cc:740] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22a807342d864d3b97a1a97e92484b22, State: Initialized, Role: FOLLOWER
21:00:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.434661 29068 consensus_queue.cc:260] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } }
21:00:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.435513 29070 consensus_queue.cc:260] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.436151 29042 heartbeater.cc:499] Master 127.27.165.60:39093 was elected leader, sending a full tablet report...
21:00:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.437425 29070 ts_tablet_manager.cc:1434] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22: Time spent starting tablet: real 0.008s user 0.005s sys 0.005s
21:00:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.437885 29070 tablet_bootstrap.cc:492] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22: Bootstrap starting.
21:00:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.439328 29069 raft_consensus.cc:359] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.439934 29070 tablet_bootstrap.cc:654] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22: Neither blocks nor log segments found. Creating new log.
21:00:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.439985 29069 raft_consensus.cc:385] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.440304 29069 raft_consensus.cc:740] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 07a12c4fa3e24352a99078a970f5d452, State: Initialized, Role: FOLLOWER
21:00:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.441200 29069 consensus_queue.cc:260] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:25.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.448457 29070 tablet_bootstrap.cc:492] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22: No bootstrap required, opened a new log
21:00:25.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.448678 29070 ts_tablet_manager.cc:1403] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.007s
21:00:25.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.449508 29070 raft_consensus.cc:359] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } }
21:00:25.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.449678 29070 raft_consensus.cc:385] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:25.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.449765 29070 raft_consensus.cc:740] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22a807342d864d3b97a1a97e92484b22, State: Initialized, Role: FOLLOWER
21:00:25.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.450004 29070 consensus_queue.cc:260] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } }
21:00:25.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.450575 29070 ts_tablet_manager.cc:1434] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
21:00:25.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.457762 29068 ts_tablet_manager.cc:1434] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61: Time spent starting tablet: real 0.032s user 0.013s sys 0.000s
21:00:25.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.461971 29068 tablet_bootstrap.cc:492] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61: Bootstrap starting.
21:00:25.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.463605 28900 heartbeater.cc:499] Master 127.27.165.60:39093 was elected leader, sending a full tablet report...
21:00:25.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.465873 29069 ts_tablet_manager.cc:1434] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452: Time spent starting tablet: real 0.037s user 0.016s sys 0.018s
21:00:25.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.466694 29069 tablet_bootstrap.cc:492] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452: Bootstrap starting.
21:00:25.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.469005 29069 tablet_bootstrap.cc:654] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452: Neither blocks nor log segments found. Creating new log.
21:00:25.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.469807 29068 tablet_bootstrap.cc:654] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61: Neither blocks nor log segments found. Creating new log.
21:00:25.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.471403 29069 tablet_bootstrap.cc:492] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452: No bootstrap required, opened a new log
21:00:25.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.471585 29069 ts_tablet_manager.cc:1403] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.004s
21:00:25.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.472322 29069 raft_consensus.cc:359] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } }
21:00:25.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.472499 29069 raft_consensus.cc:385] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:25.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.472587 29069 raft_consensus.cc:740] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 07a12c4fa3e24352a99078a970f5d452, State: Initialized, Role: FOLLOWER
21:00:25.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.472826 29069 consensus_queue.cc:260] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } }
21:00:25.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.473397 29069 ts_tablet_manager.cc:1434] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
21:00:25.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.475579 29045 tablet.cc:2378] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:00:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.476686 29068 tablet_bootstrap.cc:492] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61: No bootstrap required, opened a new log
21:00:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.476887 29068 ts_tablet_manager.cc:1403] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.004s
21:00:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.477886 29068 raft_consensus.cc:359] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.478089 29068 raft_consensus.cc:385] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.478180 29068 raft_consensus.cc:740] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd3b32698ab94b7f9e9be9c587edfb61, State: Initialized, Role: FOLLOWER
21:00:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.478441 29068 consensus_queue.cc:260] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.479766 29068 ts_tablet_manager.cc:1434] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
21:00:25.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.582011 29075 raft_consensus.cc:493] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:00:25.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.582290 29075 raft_consensus.cc:515] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } }
21:00:25.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.584236 29075 leader_election.cc:290] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cd3b32698ab94b7f9e9be9c587edfb61 (127.27.165.3:36663), 07a12c4fa3e24352a99078a970f5d452 (127.27.165.2:42895)
21:00:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.591712 28997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "fb991d2001c245ae8f513d61a903aafd" candidate_uuid: "22a807342d864d3b97a1a97e92484b22" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" is_pre_election: true
21:00:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.591706 28855 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "fb991d2001c245ae8f513d61a903aafd" candidate_uuid: "22a807342d864d3b97a1a97e92484b22" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "07a12c4fa3e24352a99078a970f5d452" is_pre_election: true
21:00:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.592242 28997 raft_consensus.cc:2468] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 22a807342d864d3b97a1a97e92484b22 in term 0.
21:00:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.592232 28855 raft_consensus.cc:2468] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 22a807342d864d3b97a1a97e92484b22 in term 0.
21:00:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.592973 28633 leader_election.cc:304] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 07a12c4fa3e24352a99078a970f5d452, 22a807342d864d3b97a1a97e92484b22, cd3b32698ab94b7f9e9be9c587edfb61; no voters:
21:00:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.593472 29075 raft_consensus.cc:2804] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:00:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.593647 29075 raft_consensus.cc:493] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:00:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.593765 29075 raft_consensus.cc:3060] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Advancing to term 1
21:00:25.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.595969 29075 raft_consensus.cc:515] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } }
21:00:25.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.596709 29075 leader_election.cc:290] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [CANDIDATE]: Term 1 election: Requested vote from peers cd3b32698ab94b7f9e9be9c587edfb61 (127.27.165.3:36663), 07a12c4fa3e24352a99078a970f5d452 (127.27.165.2:42895)
21:00:25.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.597437 28997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "fb991d2001c245ae8f513d61a903aafd" candidate_uuid: "22a807342d864d3b97a1a97e92484b22" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd3b32698ab94b7f9e9be9c587edfb61"
21:00:25.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.597437 28855 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "fb991d2001c245ae8f513d61a903aafd" candidate_uuid: "22a807342d864d3b97a1a97e92484b22" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "07a12c4fa3e24352a99078a970f5d452"
21:00:25.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.597678 28997 raft_consensus.cc:3060] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Advancing to term 1
21:00:25.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.597678 28855 raft_consensus.cc:3060] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Advancing to term 1
21:00:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.601007 28855 raft_consensus.cc:2468] T fb991d2001c245ae8f513d61a903aafd P 07a12c4fa3e24352a99078a970f5d452 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 22a807342d864d3b97a1a97e92484b22 in term 1.
21:00:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.601007 28997 raft_consensus.cc:2468] T fb991d2001c245ae8f513d61a903aafd P cd3b32698ab94b7f9e9be9c587edfb61 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 22a807342d864d3b97a1a97e92484b22 in term 1.
21:00:25.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.601671 28633 leader_election.cc:304] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 07a12c4fa3e24352a99078a970f5d452, 22a807342d864d3b97a1a97e92484b22, cd3b32698ab94b7f9e9be9c587edfb61; no voters:
21:00:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.602241 29075 raft_consensus.cc:2804] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [term 1 FOLLOWER]: Leader election won for term 1
21:00:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.602882 29075 raft_consensus.cc:697] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [term 1 LEADER]: Becoming Leader. State: Replica: 22a807342d864d3b97a1a97e92484b22, State: Running, Role: LEADER
21:00:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.603785 29075 consensus_queue.cc:237] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } }
21:00:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.609263 28512 catalog_manager.cc:5654] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 reported cstate change: term changed from 0 to 1, leader changed from <none> to 22a807342d864d3b97a1a97e92484b22 (127.27.165.1). New cstate: current_term: 1 leader_uuid: "22a807342d864d3b97a1a97e92484b22" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } health_report { overall_health: UNKNOWN } } }
21:00:25.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.640712 28760 tablet.cc:2378] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:00:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:25.673651 28903 tablet.cc:2378] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:00:25.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.793184 29077 raft_consensus.cc:493] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:00:25.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.793394 29077 raft_consensus.cc:515] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.795131 29077 leader_election.cc:290] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 22a807342d864d3b97a1a97e92484b22 (127.27.165.1:45079), cd3b32698ab94b7f9e9be9c587edfb61 (127.27.165.3:36663)
21:00:25.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.803397 28997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "5ff37ea97c994540928e18315264109d" candidate_uuid: "07a12c4fa3e24352a99078a970f5d452" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" is_pre_election: true
21:00:25.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.803444 28712 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "5ff37ea97c994540928e18315264109d" candidate_uuid: "07a12c4fa3e24352a99078a970f5d452" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22a807342d864d3b97a1a97e92484b22" is_pre_election: true
21:00:25.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.803656 28997 raft_consensus.cc:2468] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 07a12c4fa3e24352a99078a970f5d452 in term 0.
21:00:25.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.803853 28712 raft_consensus.cc:2468] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 07a12c4fa3e24352a99078a970f5d452 in term 0.
21:00:25.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.804133 28778 leader_election.cc:304] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 07a12c4fa3e24352a99078a970f5d452, cd3b32698ab94b7f9e9be9c587edfb61; no voters:
21:00:25.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.804530 29077 raft_consensus.cc:2804] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:00:25.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.804662 29077 raft_consensus.cc:493] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:00:25.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.804777 29077 raft_consensus.cc:3060] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Advancing to term 1
21:00:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.807194 29077 raft_consensus.cc:515] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:25.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.808157 29077 leader_election.cc:290] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [CANDIDATE]: Term 1 election: Requested vote from peers 22a807342d864d3b97a1a97e92484b22 (127.27.165.1:45079), cd3b32698ab94b7f9e9be9c587edfb61 (127.27.165.3:36663)
21:00:25.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.808753 28712 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "5ff37ea97c994540928e18315264109d" candidate_uuid: "07a12c4fa3e24352a99078a970f5d452" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22a807342d864d3b97a1a97e92484b22"
21:00:25.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.808826 28997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "5ff37ea97c994540928e18315264109d" candidate_uuid: "07a12c4fa3e24352a99078a970f5d452" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd3b32698ab94b7f9e9be9c587edfb61"
21:00:25.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.808967 28712 raft_consensus.cc:3060] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Advancing to term 1
21:00:25.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.809026 28997 raft_consensus.cc:3060] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Advancing to term 1
21:00:25.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.810992 28997 raft_consensus.cc:2468] T 5ff37ea97c994540928e18315264109d P cd3b32698ab94b7f9e9be9c587edfb61 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 07a12c4fa3e24352a99078a970f5d452 in term 1.
21:00:25.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.811167 28712 raft_consensus.cc:2468] T 5ff37ea97c994540928e18315264109d P 22a807342d864d3b97a1a97e92484b22 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 07a12c4fa3e24352a99078a970f5d452 in term 1.
21:00:25.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.811568 28778 leader_election.cc:304] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 07a12c4fa3e24352a99078a970f5d452, cd3b32698ab94b7f9e9be9c587edfb61; no voters:
21:00:25.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.811921 29077 raft_consensus.cc:2804] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [term 1 FOLLOWER]: Leader election won for term 1
21:00:25.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.812419 29077 raft_consensus.cc:697] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [term 1 LEADER]: Becoming Leader. State: Replica: 07a12c4fa3e24352a99078a970f5d452, State: Running, Role: LEADER
21:00:25.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.813011 29077 consensus_queue.cc:237] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:25.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:25.819188 28512 catalog_manager.cc:5654] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 reported cstate change: term changed from 0 to 1, leader changed from <none> to 07a12c4fa3e24352a99078a970f5d452 (127.27.165.2). New cstate: current_term: 1 leader_uuid: "07a12c4fa3e24352a99078a970f5d452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } health_report { overall_health: UNKNOWN } } }
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.018217 28512 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54756:
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:00:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.019657 28512 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:00:26.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.032510 28834 tablet_service.cc:1505] Processing CreateTablet for tablet a8ec98a99d1e44be88e124ce87018503 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d7bff573a7d44f1eb77c78f599d5a003]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:00:26.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.033126 28977 tablet_service.cc:1505] Processing CreateTablet for tablet a8ec98a99d1e44be88e124ce87018503 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d7bff573a7d44f1eb77c78f599d5a003]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:00:26.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.033638 28834 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a8ec98a99d1e44be88e124ce87018503. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:26.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.033948 28977 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a8ec98a99d1e44be88e124ce87018503. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:26.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.039275 28682 tablet_service.cc:1505] Processing CreateTablet for tablet a8ec98a99d1e44be88e124ce87018503 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d7bff573a7d44f1eb77c78f599d5a003]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:00:26.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.040468 28682 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a8ec98a99d1e44be88e124ce87018503. 1 dirs total, 0 dirs full, 0 dirs failed
21:00:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.048935 29099 tablet_bootstrap.cc:492] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61: Bootstrap starting.
21:00:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.049078 29100 tablet_bootstrap.cc:492] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452: Bootstrap starting.
21:00:26.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.051811 29100 tablet_bootstrap.cc:654] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452: Neither blocks nor log segments found. Creating new log.
21:00:26.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.053036 29099 tablet_bootstrap.cc:654] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61: Neither blocks nor log segments found. Creating new log.
21:00:26.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.054358 29101 tablet_bootstrap.cc:492] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: Bootstrap starting.
21:00:26.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.058274 29101 tablet_bootstrap.cc:654] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: Neither blocks nor log segments found. Creating new log.
21:00:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.059553 29099 tablet_bootstrap.cc:492] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61: No bootstrap required, opened a new log
21:00:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.059804 29099 ts_tablet_manager.cc:1403] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.005s
21:00:26.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.060585 29101 tablet_bootstrap.cc:492] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: No bootstrap required, opened a new log
21:00:26.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.060760 29101 ts_tablet_manager.cc:1403] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: Time spent bootstrapping tablet: real 0.007s user 0.004s sys 0.000s
21:00:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.060861 29099 raft_consensus.cc:359] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.061089 29099 raft_consensus.cc:385] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.061193 29100 tablet_bootstrap.cc:492] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452: No bootstrap required, opened a new log
21:00:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.061199 29099 raft_consensus.cc:740] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cd3b32698ab94b7f9e9be9c587edfb61, State: Initialized, Role: FOLLOWER
21:00:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.061363 29100 ts_tablet_manager.cc:1403] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452: Time spent bootstrapping tablet: real 0.012s user 0.006s sys 0.003s
21:00:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.061534 29099 consensus_queue.cc:260] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.061609 29101 raft_consensus.cc:359] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.061775 29101 raft_consensus.cc:385] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.061869 29101 raft_consensus.cc:740] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22a807342d864d3b97a1a97e92484b22, State: Initialized, Role: FOLLOWER
21:00:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.062144 29101 consensus_queue.cc:260] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.062706 29101 ts_tablet_manager.cc:1434] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
21:00:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.062906 29075 tablet_replica.cc:442] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } }
21:00:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.063261 29075 tablet_replica.cc:445] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: This TxnStatusTablet replica's current role is: FOLLOWER
21:00:26.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.063728 29100 raft_consensus.cc:359] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:26.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.067834 29100 raft_consensus.cc:385] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:00:26.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.067975 29100 raft_consensus.cc:740] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 07a12c4fa3e24352a99078a970f5d452, State: Initialized, Role: FOLLOWER
21:00:26.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.068691 29099 ts_tablet_manager.cc:1434] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61: Time spent starting tablet: real 0.009s user 0.000s sys 0.008s
21:00:26.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.069080 29100 consensus_queue.cc:260] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:26.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.069628 29100 ts_tablet_manager.cc:1434] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452: Time spent starting tablet: real 0.008s user 0.002s sys 0.001s
21:00:26.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.069679 29077 tablet_replica.cc:442] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } }
21:00:26.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.069957 29077 tablet_replica.cc:445] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452: This TxnStatusTablet replica's current role is: FOLLOWER
21:00:26.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.073261 29105 tablet_replica.cc:442] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } }
21:00:26.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.073700 29105 tablet_replica.cc:445] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61: This TxnStatusTablet replica's current role is: FOLLOWER
21:00:26.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.074783 29075 raft_consensus.cc:493] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:00:26.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.074997 29075 raft_consensus.cc:515] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:26.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.075737 29075 leader_election.cc:290] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 07a12c4fa3e24352a99078a970f5d452 (127.27.165.2:42895), cd3b32698ab94b7f9e9be9c587edfb61 (127.27.165.3:36663)
21:00:26.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.076268 28855 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "a8ec98a99d1e44be88e124ce87018503" candidate_uuid: "22a807342d864d3b97a1a97e92484b22" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "07a12c4fa3e24352a99078a970f5d452" is_pre_election: true
21:00:26.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.076397 28997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "a8ec98a99d1e44be88e124ce87018503" candidate_uuid: "22a807342d864d3b97a1a97e92484b22" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" is_pre_election: true
21:00:26.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.076493 28855 raft_consensus.cc:2468] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 22a807342d864d3b97a1a97e92484b22 in term 0.
21:00:26.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.076632 28997 raft_consensus.cc:2468] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 22a807342d864d3b97a1a97e92484b22 in term 0.
21:00:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.076988 28635 leader_election.cc:304] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 07a12c4fa3e24352a99078a970f5d452, 22a807342d864d3b97a1a97e92484b22; no voters:
21:00:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.077337 29075 raft_consensus.cc:2804] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:00:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.077467 29075 raft_consensus.cc:493] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:00:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.077594 29075 raft_consensus.cc:3060] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [term 0 FOLLOWER]: Advancing to term 1
21:00:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.079689 29075 raft_consensus.cc:515] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.080308 29075 leader_election.cc:290] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [CANDIDATE]: Term 1 election: Requested vote from peers 07a12c4fa3e24352a99078a970f5d452 (127.27.165.2:42895), cd3b32698ab94b7f9e9be9c587edfb61 (127.27.165.3:36663)
21:00:26.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.080976 28997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "a8ec98a99d1e44be88e124ce87018503" candidate_uuid: "22a807342d864d3b97a1a97e92484b22" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cd3b32698ab94b7f9e9be9c587edfb61"
21:00:26.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.081058 28855 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "a8ec98a99d1e44be88e124ce87018503" candidate_uuid: "22a807342d864d3b97a1a97e92484b22" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "07a12c4fa3e24352a99078a970f5d452"
21:00:26.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.081205 28997 raft_consensus.cc:3060] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61 [term 0 FOLLOWER]: Advancing to term 1
21:00:26.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.081278 28855 raft_consensus.cc:3060] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452 [term 0 FOLLOWER]: Advancing to term 1
21:00:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.082852 29082 consensus_queue.cc:1048] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:00:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.083591 28855 raft_consensus.cc:2468] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 22a807342d864d3b97a1a97e92484b22 in term 1.
21:00:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.083799 28997 raft_consensus.cc:2468] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 22a807342d864d3b97a1a97e92484b22 in term 1.
21:00:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.084127 28635 leader_election.cc:304] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 07a12c4fa3e24352a99078a970f5d452, 22a807342d864d3b97a1a97e92484b22; no voters:
21:00:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.084514 29082 raft_consensus.cc:2804] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [term 1 FOLLOWER]: Leader election won for term 1
21:00:26.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.084682 29082 raft_consensus.cc:697] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [term 1 LEADER]: Becoming Leader. State: Replica: 22a807342d864d3b97a1a97e92484b22, State: Running, Role: LEADER
21:00:26.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.084961 29082 consensus_queue.cc:237] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } }
21:00:26.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.093268 28512 catalog_manager.cc:5654] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 reported cstate change: term changed from 0 to 1, leader changed from <none> to 22a807342d864d3b97a1a97e92484b22 (127.27.165.1). New cstate: current_term: 1 leader_uuid: "22a807342d864d3b97a1a97e92484b22" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } health_report { overall_health: UNKNOWN } } }
21:00:26.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.094333 29075 tablet_replica.cc:442] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: TxnStatusTablet state changed. Reason: New leader 22a807342d864d3b97a1a97e92484b22. Latest consensus state: current_term: 1 leader_uuid: "22a807342d864d3b97a1a97e92484b22" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } }
21:00:26.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.094584 29075 tablet_replica.cc:445] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: This TxnStatusTablet replica's current role is: LEADER
21:00:26.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.099527 29112 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
21:00:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.099951 29112 txn_status_manager.cc:930] Loading transaction status metadata into memory...
21:00:26.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.107643 29075 consensus_queue.cc:1048] T fb991d2001c245ae8f513d61a903aafd P 22a807342d864d3b97a1a97e92484b22 [LEADER]: Connected to new peer: Peer: permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:00:26.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.141741 28855 raft_consensus.cc:1275] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452 [term 1 FOLLOWER]: Refusing update from remote peer 22a807342d864d3b97a1a97e92484b22: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:00:26.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.141742 28996 raft_consensus.cc:1275] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61 [term 1 FOLLOWER]: Refusing update from remote peer 22a807342d864d3b97a1a97e92484b22: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:00:26.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.142566 29075 consensus_queue.cc:1048] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [LEADER]: Connected to new peer: Peer: permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:00:26.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.143734 29075 consensus_queue.cc:1048] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:00:26.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.157817 29077 tablet_replica.cc:442] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452: TxnStatusTablet state changed. Reason: New leader 22a807342d864d3b97a1a97e92484b22. Latest consensus state: current_term: 1 leader_uuid: "22a807342d864d3b97a1a97e92484b22" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } }
21:00:26.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.158116 29077 tablet_replica.cc:445] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452: This TxnStatusTablet replica's current role is: FOLLOWER
21:00:26.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.166710 29105 tablet_replica.cc:442] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61: TxnStatusTablet state changed. Reason: New leader 22a807342d864d3b97a1a97e92484b22. Latest consensus state: current_term: 1 leader_uuid: "22a807342d864d3b97a1a97e92484b22" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } }
21:00:26.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.167024 29105 tablet_replica.cc:445] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61: This TxnStatusTablet replica's current role is: FOLLOWER
21:00:26.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.168869 29110 mvcc.cc:204] Tried to move back new op lower bound from 7230726251066617856 to 7230726250847227904. Current Snapshot: MvccSnapshot[applied={T|T < 7230726251066617856}]
21:00:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.177266 29105 tablet_replica.cc:442] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "22a807342d864d3b97a1a97e92484b22" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } }
21:00:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.177629 29105 tablet_replica.cc:445] T a8ec98a99d1e44be88e124ce87018503 P cd3b32698ab94b7f9e9be9c587edfb61: This TxnStatusTablet replica's current role is: FOLLOWER
21:00:26.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.180402 29075 tablet_replica.cc:442] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "22a807342d864d3b97a1a97e92484b22" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } }
21:00:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.180634 29075 tablet_replica.cc:445] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: This TxnStatusTablet replica's current role is: LEADER
21:00:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.180958 29075 tablet_replica.cc:442] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "22a807342d864d3b97a1a97e92484b22" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } }
21:00:26.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.181142 29075 tablet_replica.cc:445] T a8ec98a99d1e44be88e124ce87018503 P 22a807342d864d3b97a1a97e92484b22: This TxnStatusTablet replica's current role is: LEADER
21:00:26.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.186764 29077 tablet_replica.cc:442] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "22a807342d864d3b97a1a97e92484b22" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 } } peers { permanent_uuid: "07a12c4fa3e24352a99078a970f5d452" member_type: VOTER last_known_addr { host: "127.27.165.2" port: 42895 } } peers { permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 } } }
21:00:26.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.187041 29077 tablet_replica.cc:445] T a8ec98a99d1e44be88e124ce87018503 P 07a12c4fa3e24352a99078a970f5d452: This TxnStatusTablet replica's current role is: FOLLOWER
21:00:26.187 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
21:00:26.190 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 99e1ceca529848e3ade6b45912eec9e9
21:00:26.192 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet fb991d2001c245ae8f513d61a903aafd for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
21:00:26.193 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5ff37ea97c994540928e18315264109d for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
21:00:26.193 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=fb991d2001c245ae8f513d61a903aafd@[07a12c4fa3e24352a99078a970f5d452(127.27.165.2:42895),22a807342d864d3b97a1a97e92484b22(127.27.165.1:45079)[L],cd3b32698ab94b7f9e9be9c587edfb61(127.27.165.3:36663)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=5ff37ea97c994540928e18315264109d@[07a12c4fa3e24352a99078a970f5d452(127.27.165.2:42895)[L],22a807342d864d3b97a1a97e92484b22(127.27.165.1:45079),cd3b32698ab94b7f9e9be9c587edfb61(127.27.165.3:36663)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
21:00:26.193 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.165.60:39093, [2ms] Sub RPC GetTableLocations: received response from server master-127.27.165.60:39093: OK, deferred=null) after lookup
21:00:26.194 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 22a807342d864d3b97a1a97e92484b22(127.27.165.1:45079)] connecting to peer
21:00:26.195 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 22a807342d864d3b97a1a97e92484b22(127.27.165.1:45079)] Successfully connected to peer
21:00:26.208 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xe71082c8, L:/127.0.0.1:47118 - R:/127.27.165.1:45079] using TOKEN/null
21:00:26.261 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.165.60:39093
21:00:26.263 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:00:26.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.263998 28308 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu pid 28478
21:00:26.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.265542 28552 generic_service.cc:196] Checking for leaks (request via RPC)
21:00:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.391198 28308 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu with pid 28478
21:00:26.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.397874 29077 consensus_queue.cc:1048] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cd3b32698ab94b7f9e9be9c587edfb61" member_type: VOTER last_known_addr { host: "127.27.165.3" port: 36663 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:00:26.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.414131 29077 consensus_queue.cc:1048] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22a807342d864d3b97a1a97e92484b22" member_type: VOTER last_known_addr { host: "127.27.165.1" port: 45079 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:00:26.422 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.165.60:39093(127.27.165.60:39093)] handling channelInactive
21:00:26.422 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state READY due to: connection closed
21:00:26.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.422658 28780 connection.cc:537] client connection to 127.27.165.60:39093 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:00:26.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.422842 28922 connection.cc:537] client connection to 127.27.165.60:39093 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:00:26.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.423568 29042 heartbeater.cc:646] Failed to heartbeat to 127.27.165.60:39093 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:00:26.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.424257 28900 heartbeater.cc:646] Failed to heartbeat to 127.27.165.60:39093 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:00:26.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.424890 28635 connection.cc:537] client connection to 127.27.165.60:39093 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:00:26.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.425491 28757 heartbeater.cc:646] Failed to heartbeat to 127.27.165.60:39093 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:00:26.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:00:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.429092 29130 negotiation.cc:337] Failed RPC negotiation. Trace:
21:00:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1209 21:00:26.424618 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.27.165.60:39093 (local address 127.27.165.3:34485)
21:00:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1209 21:00:26.427554 (+ 2936us) negotiation.cc:107] Waiting for socket to connect
21:00:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1209 21:00:26.427579 (+ 25us) client_negotiation.cc:174] Beginning negotiation
21:00:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1209 21:00:26.427749 (+ 170us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
21:00:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1209 21:00:26.427992 (+ 243us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: BlockingWrite error: write error: Connection reset by peer (error 104)
21:00:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":2286,"thread_start_us":2103,"threads_started":1}
21:00:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.429956 29131 negotiation.cc:337] Failed RPC negotiation. Trace:
21:00:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1209 21:00:26.426128 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.27.165.60:39093 (local address 127.27.165.1:39133)
21:00:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1209 21:00:26.428502 (+ 2374us) negotiation.cc:107] Waiting for socket to connect
21:00:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1209 21:00:26.428832 (+ 330us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection reset by peer (error 104)
21:00:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":2256,"thread_start_us":1998,"threads_started":1}
21:00:26.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.691929 29132 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 151e9e3eb2c046a4b4715f63e74ec358)
21:00:26.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.692169 29132 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:26.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.692541 29133 raft_consensus.cc:493] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 151e9e3eb2c046a4b4715f63e74ec358)
21:00:26.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.692726 29133 raft_consensus.cc:515] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.693755 29132 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:26.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.693975 29133 leader_election.cc:290] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers da910329c4c84f5f8faa465510be0b11 (127.27.165.61:46277), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:26.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.694455 28359 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "da910329c4c84f5f8faa465510be0b11" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "466af8d834514b7b9ee9f24cdf09b088" is_pre_election: true
21:00:26.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.694753 28359 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate da910329c4c84f5f8faa465510be0b11 in term 1.
21:00:26.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.694969 28441 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "466af8d834514b7b9ee9f24cdf09b088" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "da910329c4c84f5f8faa465510be0b11" is_pre_election: true
21:00:26.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.695207 28441 raft_consensus.cc:2468] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466af8d834514b7b9ee9f24cdf09b088 in term 1.
21:00:26.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.696144 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 466af8d834514b7b9ee9f24cdf09b088, da910329c4c84f5f8faa465510be0b11; no voters:
21:00:26.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.696592 29132 raft_consensus.cc:2804] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:00:26.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.696749 29132 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 151e9e3eb2c046a4b4715f63e74ec358)
21:00:26.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.696859 29132 raft_consensus.cc:3060] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 1 FOLLOWER]: Advancing to term 2
21:00:26.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.696838 28337 leader_election.cc:304] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 466af8d834514b7b9ee9f24cdf09b088, da910329c4c84f5f8faa465510be0b11; no voters:
21:00:26.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.697347 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:26.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.697553 29133 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:00:26.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.697680 29133 raft_consensus.cc:493] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 151e9e3eb2c046a4b4715f63e74ec358)
21:00:26.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.697768 29133 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 1 FOLLOWER]: Advancing to term 2
21:00:26.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.698678 28336 leader_election.cc:336] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:26.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.699771 29132 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:26.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.700420 29132 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 2 election: Requested vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:26.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.700791 29133 raft_consensus.cc:515] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:26.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.701226 28359 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "da910329c4c84f5f8faa465510be0b11" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "466af8d834514b7b9ee9f24cdf09b088"
21:00:26.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.701522 28359 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate da910329c4c84f5f8faa465510be0b11 in current term 2: Already voted for candidate 466af8d834514b7b9ee9f24cdf09b088 in this term.
21:00:26.701 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.702024 29133 leader_election.cc:290] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 2 election: Requested vote from peers da910329c4c84f5f8faa465510be0b11 (127.27.165.61:46277), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:26.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.702188 28441 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "466af8d834514b7b9ee9f24cdf09b088" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "da910329c4c84f5f8faa465510be0b11"
21:00:26.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.702472 28441 raft_consensus.cc:2393] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 466af8d834514b7b9ee9f24cdf09b088 in current term 2: Already voted for candidate da910329c4c84f5f8faa465510be0b11 in this term.
21:00:26.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.702757 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:26.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.703174 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:26.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.703651 29132 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:00:26.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:26.703975 28336 leader_election.cc:336] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:26.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.704102 28336 leader_election.cc:304] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 466af8d834514b7b9ee9f24cdf09b088; no voters: 151e9e3eb2c046a4b4715f63e74ec358, da910329c4c84f5f8faa465510be0b11
21:00:26.704 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:26.704581 29133 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:00:26.704 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.704 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.27.165.60:39093(127.27.165.60:39093) for tablet Kudu Master: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.716 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.717 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.719 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:26.736 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.737 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.737 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.739 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:26.756 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.759 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:26.776 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.777 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.779 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:26.796 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.799 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:26.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.819 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:26.896 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.897 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.897 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:26.916 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.918 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.919 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:26.920 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.27.165.60:39093, [7ms] received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [7ms] delaying RPC due to: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [21ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [21ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [62ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [101ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [102ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [121ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [200ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [201ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [201ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [221ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [222ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, deferred=Deferred@367401347(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@7e66eb6a -> (continuation of Deferred@1821247968 after retry RPC after error@1741741464) -> (continuation of Deferred@1113586400 after retry RPC after error@195781754) -> (continuation of Deferred@402042342 after retry RPC after error@1982570141) -> (continuation of Deferred@1268703810 after retry RPC after error@457627647) -> (continuation of Deferred@972935404 after retry RPC after error@319323704) -> (continuation of Deferred@792199913 after retry RPC after error@49070558) -> (continuation of Deferred@1430330335 after retry RPC after error@99112689), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@6aa833ef -> (continuation of Deferred@1821247968 after retry RPC after error@1741741464) -> (continuation of Deferred@1113586400 after retry RPC after error@195781754) -> (continuation of Deferred@402042342 after retry RPC after error@1982570141) -> (continuation of Deferred@1268703810 after retry RPC after error@457627647) -> (continuation of Deferred@972935404 after retry RPC after error@319323704) -> (continuation of Deferred@792199913 after retry RPC after error@49070558) -> (continuation of Deferred@1430330335 after retry RPC after error@99112689)))
21:00:26.921 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.27.165.60:39093, [7ms] received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [7ms] delaying RPC due to: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [21ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [21ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [62ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [101ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [102ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [121ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [200ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [201ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [201ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [221ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [222ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, deferred=Deferred@367401347(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@7e66eb6a -> (continuation of Deferred@1821247968 after retry RPC after error@1741741464) -> (continuation of Deferred@1113586400 after retry RPC after error@195781754) -> (continuation of Deferred@402042342 after retry RPC after error@1982570141) -> (continuation of Deferred@1268703810 after retry RPC after error@457627647) -> (continuation of Deferred@972935404 after retry RPC after error@319323704) -> (continuation of Deferred@792199913 after retry RPC after error@49070558) -> (continuation of Deferred@1430330335 after retry RPC after error@99112689), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@6aa833ef -> (continuation of Deferred@1821247968 after retry RPC after error@1741741464) -> (continuation of Deferred@1113586400 after retry RPC after error@195781754) -> (continuation of Deferred@402042342 after retry RPC after error@1982570141) -> (continuation of Deferred@1268703810 after retry RPC after error@457627647) -> (continuation of Deferred@972935404 after retry RPC after error@319323704) -> (continuation of Deferred@792199913 after retry RPC after error@49070558) -> (continuation of Deferred@1430330335 after retry RPC after error@99112689)))
21:00:26.936 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.937 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.937 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:26.956 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.959 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:26.976 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.979 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:26.997 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:26.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:26.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.000 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:27.017 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:27.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.019 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:27.056 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:27.057 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.057 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.058 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:27.076 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:27.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.078 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.091661 29133 raft_consensus.cc:493] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:00:27.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.091928 29133 raft_consensus.cc:515] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:27.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.092700 29133 leader_election.cc:290] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers da910329c4c84f5f8faa465510be0b11 (127.27.165.61:46277), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:27.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:27.093374 28336 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111) [suppressed 10 similar messages]
21:00:27.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.093734 28441 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "466af8d834514b7b9ee9f24cdf09b088" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "da910329c4c84f5f8faa465510be0b11" is_pre_election: true
21:00:27.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.094022 28441 raft_consensus.cc:2468] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 466af8d834514b7b9ee9f24cdf09b088 in term 2.
21:00:27.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:27.094439 28336 leader_election.cc:336] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:27.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.094635 28337 leader_election.cc:304] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 466af8d834514b7b9ee9f24cdf09b088, da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358
21:00:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.094992 29133 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 2 FOLLOWER]: Leader pre-election won for term 3
21:00:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.095100 29133 raft_consensus.cc:493] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:00:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.095201 29133 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 2 FOLLOWER]: Advancing to term 3
21:00:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.097108 29133 raft_consensus.cc:515] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.097733 29133 leader_election.cc:290] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 3 election: Requested vote from peers da910329c4c84f5f8faa465510be0b11 (127.27.165.61:46277), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:27.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.098392 28441 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "466af8d834514b7b9ee9f24cdf09b088" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "da910329c4c84f5f8faa465510be0b11"
21:00:27.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.098639 28441 raft_consensus.cc:3060] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 2 FOLLOWER]: Advancing to term 3
21:00:27.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:27.099210 28336 leader_election.cc:336] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.100936 28441 raft_consensus.cc:2468] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 466af8d834514b7b9ee9f24cdf09b088 in term 3.
21:00:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.101498 28337 leader_election.cc:304] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 466af8d834514b7b9ee9f24cdf09b088, da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358
21:00:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.101826 29133 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 3 FOLLOWER]: Leader election won for term 3
21:00:27.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.102326 29133 raft_consensus.cc:697] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [term 3 LEADER]: Becoming Leader. State: Replica: 466af8d834514b7b9ee9f24cdf09b088, State: Running, Role: LEADER
21:00:27.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.102881 29133 consensus_queue.cc:237] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:27.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.104354 29141 sys_catalog.cc:455] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 466af8d834514b7b9ee9f24cdf09b088. Latest consensus state: current_term: 3 leader_uuid: "466af8d834514b7b9ee9f24cdf09b088" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:27.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.104614 29141 sys_catalog.cc:458] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [sys.catalog]: This master's current role is: LEADER
21:00:27.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.105280 29142 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:00:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.107887 29142 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback [id=99e1ceca529848e3ade6b45912eec9e9]
21:00:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.108417 29142 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=d7bff573a7d44f1eb77c78f599d5a003]
21:00:27.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.110271 29142 tablet_loader.cc:96] loaded metadata for tablet 5ff37ea97c994540928e18315264109d (table txn_manager_fallback [id=99e1ceca529848e3ade6b45912eec9e9])
21:00:27.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.110669 29142 tablet_loader.cc:96] loaded metadata for tablet a8ec98a99d1e44be88e124ce87018503 (table kudu_system.kudu_transactions [id=d7bff573a7d44f1eb77c78f599d5a003])
21:00:27.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.111099 29142 tablet_loader.cc:96] loaded metadata for tablet fb991d2001c245ae8f513d61a903aafd (table txn_manager_fallback [id=99e1ceca529848e3ade6b45912eec9e9])
21:00:27.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.111366 29142 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:00:27.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.112351 29142 catalog_manager.cc:1269] Loaded cluster ID: d31db4baae2b4fe98542ff4d78378ffb
21:00:27.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.112443 29142 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:00:27.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.113660 29142 catalog_manager.cc:1514] Loading token signing keys...
21:00:27.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.114580 29142 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088: Loaded TSK: 0
21:00:27.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.115588 29142 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:00:27.177 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:27.178 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.178 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:27.181 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:00:27.182 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=245), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [1ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [2ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [22ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [23ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [42ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [43ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [62ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [63ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [82ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [83ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [142ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [143ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [242ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [243ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [244ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, deferred=Deferred@1351309087(state=PENDING, result=null, callback=(continuation of Deferred@2126499115 after retry RPC after error@506155651) -> (continuation of Deferred@1391768969 after retry RPC after error@1481450343) -> (continuation of Deferred@1478196176 after retry RPC after error@484476186) -> (continuation of Deferred@1800348658 after retry RPC after error@1520989159) -> (continuation of Deferred@1508787865 after retry RPC after error@1094631659) -> (continuation of Deferred@1409039245 after retry RPC after error@1255296350) -> (continuation of Deferred@1638076435 after retry RPC after error@1454222676), errback=(continuation of Deferred@2126499115 after retry RPC after error@506155651) -> (continuation of Deferred@1391768969 after retry RPC after error@1481450343) -> (continuation of Deferred@1478196176 after retry RPC after error@484476186) -> (continuation of Deferred@1800348658 after retry RPC after error@1520989159) -> (continuation of Deferred@1508787865 after retry RPC after error@1094631659) -> (continuation of Deferred@1409039245 after retry RPC after error@1255296350) -> (continuation of Deferred@1638076435 after retry RPC after error@1454222676))) after lookup
21:00:27.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.193291 28441 raft_consensus.cc:1275] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Refusing update from remote peer 466af8d834514b7b9ee9f24cdf09b088: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
21:00:27.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.194111 29141 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [LEADER]: Connected to new peer: Peer: permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
21:00:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.196264 29132 sys_catalog.cc:455] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 466af8d834514b7b9ee9f24cdf09b088. Latest consensus state: current_term: 3 leader_uuid: "466af8d834514b7b9ee9f24cdf09b088" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.196589 29132 sys_catalog.cc:458] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [sys.catalog]: This master's current role is: FOLLOWER
21:00:27.197 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [1ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [2ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [22ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [23ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [42ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [43ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [62ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [63ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [82ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [83ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [142ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [143ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [242ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [243ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [244ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: OK, [246ms] sending RPC to server master-127.27.165.62:41881, deferred=Deferred@1351309087(state=PENDING, result=null, callback=(continuation of Deferred@2126499115 after retry RPC after error@506155651) -> (continuation of Deferred@1391768969 after retry RPC after error@1481450343) -> (continuation of Deferred@1478196176 after retry RPC after error@484476186) -> (continuation of Deferred@1800348658 after retry RPC after error@1520989159) -> (continuation of Deferred@1508787865 after retry RPC after error@1094631659) -> (continuation of Deferred@1409039245 after retry RPC after error@1255296350) -> (continuation of Deferred@1638076435 after retry RPC after error@1454222676) -> (continuation of Deferred@59187662 after retry RPC@102443684), errback=(continuation of Deferred@2126499115 after retry RPC after error@506155651) -> (continuation of Deferred@1391768969 after retry RPC after error@1481450343) -> (continuation of Deferred@1478196176 after retry RPC after error@484476186) -> (continuation of Deferred@1800348658 after retry RPC after error@1520989159) -> (continuation of Deferred@1508787865 after retry RPC after error@1094631659) -> (continuation of Deferred@1409039245 after retry RPC after error@1255296350) -> (continuation of Deferred@1638076435 after retry RPC after error@1454222676) -> (continuation of Deferred@59187662 after retry RPC@102443684)))
21:00:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.201366 29132 sys_catalog.cc:455] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "466af8d834514b7b9ee9f24cdf09b088" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.201767 29132 sys_catalog.cc:458] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [sys.catalog]: This master's current role is: FOLLOWER
21:00:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:27.203215 28380 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:50348 (request call id 17) took 20 ms (client timeout 4 ms). Trace:
21:00:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:27.203524 28380 rpcz_store.cc:269] 1209 21:00:27.183266 (+ 0us) service_pool.cc:168] Inserting onto call queue
21:00:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1209 21:00:27.183397 (+ 131us) service_pool.cc:225] Handling call
21:00:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1209 21:00:27.203103 (+ 19706us) inbound_call.cc:173] Queueing success response
21:00:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":513,"thread_start_us":353,"threads_started":2}
21:00:27.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.212153 29133 sys_catalog.cc:455] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "466af8d834514b7b9ee9f24cdf09b088" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } } }
21:00:27.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.212976 29133 sys_catalog.cc:458] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 [sys.catalog]: This master's current role is: LEADER
21:00:27.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:27.212976 28336 consensus_peers.cc:597] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 -> Peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Couldn't send request to peer 151e9e3eb2c046a4b4715f63e74ec358. Status: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:00:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28: runtime error: signed integer overflow: 670827496856 + 9223371366027279535 cannot be represented in type 'long'
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:27.265401 28350 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50366:
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:00:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:00:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:00:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:27.270952 28350 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:00:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:27.695870 28336 consensus_peers.cc:597] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 -> Peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Couldn't send request to peer 151e9e3eb2c046a4b4715f63e74ec358. Status: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:00:28.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:28.165161 28336 consensus_peers.cc:597] T 00000000000000000000000000000000 P 466af8d834514b7b9ee9f24cdf09b088 -> Peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Couldn't send request to peer 151e9e3eb2c046a4b4715f63e74ec358. Status: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
21:00:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:28.252529 28319 master_runner.cc:186] Error getting master registration for 127.27.165.60:39093: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111),
21:00:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:28.252899 28319 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:00:28.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7f194bda0008 in kudu::operator+(kudu::MonoTime const&, kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28
21:00:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0xa78cf5 in kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/countdown_latch.h:99:38
21:00:28.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7f195fafae22 in kudu::Promise<kudu::Status>::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/promise.h:57:16
21:00:28.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7f195faea934 in kudu::master::Master::WaitForTxnManagerInit(kudu::MonoTime) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/master.cc:482:48
21:00:28.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7f195fc5087e in kudu::transactions::TxnManager::CheckInitialized(kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:283:21
21:00:28.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7f195fc5145b in kudu::transactions::TxnManager::KeepTransactionAlive(long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:226:3
21:00:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7f195fc5609f in kudu::transactions::TxnManagerServiceImpl::KeepTransactionAlive(kudu::transactions::KeepTransactionAliveRequestPB const*, kudu::transactions::KeepTransactionAliveResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager_service.cc:159:42
21:00:28.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7f1959a7f2d6 in kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/asan/src/kudu/master/txn_manager.service.cc:271:13
21:00:28.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7f1959a7f005 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
21:00:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:28.415545 29043 heartbeater.cc:499] Master 127.27.165.62:41881 was elected leader, sending a full tablet report...
21:00:28.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:28.426261 28758 heartbeater.cc:499] Master 127.27.165.62:41881 was elected leader, sending a full tablet report...
21:00:28.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7f1952dea56e in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
21:00:28.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7f1952de944e in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3
21:00:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7f1952def685 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:229:15
21:00:28.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7f1952deffaf in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:92:5
21:00:28.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7f1952defdd8 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
21:00:28.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0xa8970c in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
21:00:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7f194be703f8 in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3
21:00:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7f194f47b6da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
21:00:28.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:28.455785 28692 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.27.165.60:39093)
21:00:28.456 [DEBUG - Time-limited test] (KuduTransaction.java:545) stopping keepalive heartbeating after initiating commit (txn ID 0)
21:00:28.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7f1947fac71e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12171e)
21:00:28.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:00:28.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28 in
21:00:28.483 [ERROR - kudu-nio-0] (Connection.java:452) [peer master-127.27.165.62:41881(127.27.165.62:41881)] unexpected exception from downstream on [id: 0xec866440, L:/127.0.0.1:50348 - R:/127.27.165.62:41881]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) ~[?:1.8.0_275]
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:192)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379)
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255)
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132)
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21:00:28.484 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state READY due to: [peer master-127.27.165.62:41881(127.27.165.62:41881)] unexpected exception from downstream on [id: 0xec866440, L:/127.0.0.1:50348 - R:/127.27.165.62:41881]
21:00:28.484 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.27.165.62:41881(127.27.165.62:41881) for tablet Kudu Master: [peer master-127.27.165.62:41881(127.27.165.62:41881)] unexpected exception from downstream on [id: 0xec866440, L:/127.0.0.1:50348 - R:/127.27.165.62:41881]
21:00:28.484 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.27.165.62:41881(127.27.165.62:41881) for tablet Kudu Master: [peer master-127.27.165.62:41881(127.27.165.62:41881)] unexpected exception from downstream on [id: 0xec866440, L:/127.0.0.1:50348 - R:/127.27.165.62:41881]
21:00:28.485 [DEBUG - kudu-nio-0] (RemoteTablet.java:144) tablet Kudu Master already removed ts master-127.27.165.62:41881, size left is 0
21:00:28.487 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.165.62:41881(127.27.165.62:41881)] handling channelInactive
21:00:28.497 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
21:00:28.506 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:28.506 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:28.507 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.507 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.507 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.507 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.507 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.507 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.508 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:28.516 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:28.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:28.517 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.517 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.517 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:28.536 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:28.537 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:28.537 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.537 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.537 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.539 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:28.556 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:28.557 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:28.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.558 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.558 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.559 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:28.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:28.577 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:28.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.580 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:28.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:28.617 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:28.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.619 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:28.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:28.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:28.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:28.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:28.742036 29178 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:00:28.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:28.742307 29178 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:28.743146 29178 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:28.744359 28417 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111) [suppressed 6 similar messages]
21:00:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:28.745880 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:00:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:28.746061 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:28.746142 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:28.746440 29178 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:28.816 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:28.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:28.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:28.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:28.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:28.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:28.946398 28901 heartbeater.cc:499] Master 127.27.165.62:41881 was elected leader, sending a full tablet report...
21:00:29.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:29.094857 29178 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:00:29.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:29.095075 29178 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:29.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:29.095654 29178 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:29.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:29.096976 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:00:29.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:29.097189 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:29.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:29.097295 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:29.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:29.097546 29178 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:29.296 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:29.296 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:29.297 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:29.297 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:29.297 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:29.297 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:29.297 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:29.297 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:29.298 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:29.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:29.717517 29186 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:00:29.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:29.717753 29186 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:29.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:29.718495 29186 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:29.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:29.724336 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:00:29.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:29.724613 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:29.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:29.724733 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:29.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:29.725016 29186 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:29.936 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:29.937 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:29.937 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:29.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:29.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:29.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:29.938 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:29.938 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:29.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:30.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:30.041442 29186 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:00:30.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:30.041683 29186 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:30.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:30.042418 29186 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:30.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:30.043954 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:00:30.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:30.044149 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:30.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:30.044224 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:30.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:30.044489 29186 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:31.317 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:31.318 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:31.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:31.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:31.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:31.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:31.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:31.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:31.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:31.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:31.538961 29198 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:00:31.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:31.539191 29198 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:31.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:31.539878 29198 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:31.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:31.542520 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:00:31.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:31.542773 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:31.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:31.542929 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:31.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:31.543258 29198 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:32.004148 29198 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:00:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:32.004381 29198 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:32.005133 29198 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:32.006542 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:00:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:32.006798 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:32.006892 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:32.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:32.007149 29198 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:33.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:33.531226 28694 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:33.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:33.535838 28694 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:33.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:33.537393 28692 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.27.165.62:41881)
21:00:34.096 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:34.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:34.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:34.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:34.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:34.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:34.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:34.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:34.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:35.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:35.361476 29218 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:00:35.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:35.361716 29218 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:35.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:35.362468 29218 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:35.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:35.363832 28417 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111) [suppressed 11 similar messages]
21:00:35.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:35.365334 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:00:35.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:35.365549 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:35.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:35.365634 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:35.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:35.365924 29218 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:35.436 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:35.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:35.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:35.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:35.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:35.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:35.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:35.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:35.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:36.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:36.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:36.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:36.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:36.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:36.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:36.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:36.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:36.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:37.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:37.303818 29233 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:00:37.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:37.304083 29233 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:37.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:37.304848 29233 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:37.307878 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:00:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:37.308094 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:37.308175 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:37.308461 29233 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:38.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:38.591885 28694 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:00:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:38.594295 28694 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:00:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:38.595781 28692 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.27.165.62:41881)
21:00:40.276 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:40.277 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:40.277 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:40.277 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:40.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:40.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:40.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:40.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:40.279 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:42.258154 29261 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:00:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:42.258378 29261 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:42.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:42.259191 29261 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:42.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:42.260385 28417 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111) [suppressed 3 similar messages]
21:00:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:42.261858 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:00:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:42.262055 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:42.262192 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:42.262509 29261 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:43.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:43.649909 28694 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:00:43.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:43.652302 28694 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:00:43.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:43.653831 28692 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.27.165.62:41881)
21:00:44.136 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:44.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:44.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:44.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:44.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:44.140 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:44.141 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:44.141 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:44.141 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:44.556 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:44.556 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:44.557 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:44.557 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:44.557 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:44.557 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:44.557 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:44.557 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:44.558 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:46.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:46.457 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:46.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:46.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:46.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:46.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:46.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:46.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:46.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:47.977 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:47.978 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:47.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:47.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:47.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:47.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:47.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:47.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:47.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:48.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:48.699035 28694 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:00:48.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:48.702458 28694 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:00:48.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:48.703984 28692 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.27.165.62:41881)
21:00:48.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:48.757 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:48.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:48.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:48.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:48.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:48.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:48.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:48.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:48.896 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:48.896 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:48.897 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:48.897 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:48.897 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:48.897 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:48.897 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:48.897 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:48.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:50.236 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:50.237 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:50.237 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:50.237 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:50.237 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:50.237 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:50.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:50.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:50.239 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:50.476 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:50.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:50.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:50.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:50.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:50.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:50.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:50.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:50.480 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:51.717 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:51.718 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:51.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:51.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:51.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:51.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:51.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:51.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:51.720 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:53.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:53.297945 29321 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:00:53.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:53.298183 29321 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:53.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:53.298971 29321 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:53.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:53.300410 28417 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111) [suppressed 1 similar messages]
21:00:53.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:53.301805 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:00:53.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:53.302024 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:53.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:53.302160 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:53.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:53.302510 29321 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:53.768813 28694 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:00:53.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:53.774226 28694 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:00:53.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:53.775681 28692 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.27.165.62:41881)
21:00:54.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:54.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:54.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:54.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:54.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:54.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:54.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:54.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:54.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:56.056 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:56.057 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:56.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:56.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:56.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:56.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:56.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:56.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:56.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:56.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:56.676067 29344 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:00:56.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:56.676275 29344 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:00:56.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:56.676976 29344 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:00:56.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:56.679138 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:00:56.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:56.679330 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:00:56.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:56.679415 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:00:56.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:00:56.679647 29344 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:00:57.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:57.097 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:57.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:57.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:57.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:57.098 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:57.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:57.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:57.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:00:58.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:58.804190 28694 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:00:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:58.805680 28694 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:00:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:00:58.807864 28692 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.27.165.62:41881)
21:00:59.516 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:00:59.517 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:00:59.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:59.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:59.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:59.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:59.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:00:59.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:00:59.519 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:01.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:01.137 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:01.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:01.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:01.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:01.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:01.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:01.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:01.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:03.156 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:03.157 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:03.157 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:03.157 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:03.158 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:03.158 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:03.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:03.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:03.158 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:03.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:03.843832 28694 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:01:03.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:03.845230 28694 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:01:03.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:03.846599 28692 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.27.165.62:41881)
21:01:05.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:05.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:05.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:05.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:05.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:05.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:05.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:05.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:05.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:06.216 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:06.217 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:06.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:06.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:06.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:06.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:06.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:06.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:06.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:06.900293 29407 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:01:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:06.900539 29407 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:01:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:06.901427 29407 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:01:06.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:06.902997 28417 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111) [suppressed 3 similar messages]
21:01:06.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:06.904464 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:01:06.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:06.904718 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:01:06.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:06.904805 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:01:06.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:06.905037 29407 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:01:08.476 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:08.477 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:08.477 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:08.477 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:08.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:08.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:08.478 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:08.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:08.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:08.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:08.876094 28694 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:01:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:08.879556 28694 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:01:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:08.881054 28692 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.27.165.62:41881)
21:01:09.436 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:09.437 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:09.437 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:09.437 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:09.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:09.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:09.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:09.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:09.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:11.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:11.117 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:11.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:11.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:11.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:11.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:11.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:11.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:11.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:12.836 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:12.837 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:12.837 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:12.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:12.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:12.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:12.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:12.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:12.839 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:13.056 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:13.056 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:13.057 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:13.057 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:13.057 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:13.057 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:13.057 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:13.057 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:13.058 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:13.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:13.344149 29451 raft_consensus.cc:493] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 466af8d834514b7b9ee9f24cdf09b088)
21:01:13.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:13.344372 29451 raft_consensus.cc:515] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "466af8d834514b7b9ee9f24cdf09b088" member_type: VOTER last_known_addr { host: "127.27.165.62" port: 41881 } } peers { permanent_uuid: "da910329c4c84f5f8faa465510be0b11" member_type: VOTER last_known_addr { host: "127.27.165.61" port: 46277 } } peers { permanent_uuid: "151e9e3eb2c046a4b4715f63e74ec358" member_type: VOTER last_known_addr { host: "127.27.165.60" port: 39093 } }
21:01:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:13.345067 29451 leader_election.cc:290] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881), 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093)
21:01:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:13.346446 28417 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111) [suppressed 1 similar messages]
21:01:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:13.347841 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 466af8d834514b7b9ee9f24cdf09b088 (127.27.165.62:41881): Network error: Client connection negotiation failed: client connection to 127.27.165.62:41881: connect: Connection refused (error 111)
21:01:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:13.348110 28417 leader_election.cc:336] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 151e9e3eb2c046a4b4715f63e74ec358 (127.27.165.60:39093): Network error: Client connection negotiation failed: client connection to 127.27.165.60:39093: connect: Connection refused (error 111)
21:01:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:13.348219 28417 leader_election.cc:304] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da910329c4c84f5f8faa465510be0b11; no voters: 151e9e3eb2c046a4b4715f63e74ec358, 466af8d834514b7b9ee9f24cdf09b088
21:01:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:13.348531 29451 raft_consensus.cc:2749] T 00000000000000000000000000000000 P da910329c4c84f5f8faa465510be0b11 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:01:13.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:13.877 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:13.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:13.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:13.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:13.878 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:13.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:13.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:13.879 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:13.929961 28694 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:01:13.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:13.933401 28694 meta_cache.cc:788] Timed out: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: 'fb991d2001c245ae8f513d61a903aafd', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.27.165.62:41881,127.27.165.61:46277,127.27.165.60:39093, num_attempts: 1)
21:01:13.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:13.935218 28692 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.27.165.62:41881)
21:01:15.636 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:15.636 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:15.637 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:15.637 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:15.637 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:15.637 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:15.637 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:15.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:15.639 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:15.816 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.60:39093(127.27.165.60:39093)] connecting to peer
21:01:15.817 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.27.165.62:41881(127.27.165.62:41881)] connecting to peer
21:01:15.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:15.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:15.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.60:39093(127.27.165.60:39093)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:15.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.165.62:41881(127.27.165.62:41881)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:15.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.60:39093: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093
21:01:15.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.165.62:41881: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
21:01:15.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881; will retry
21:01:15.820 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=Kudu Master, attempt=41, TimeoutTracker(timeout=50000, elapsed=47342), Traces: [0ms] sending RPC to server master-127.27.165.62:41881, [7ms] received response from server master-127.27.165.62:41881: Network error: [peer master-127.27.165.62:41881(127.27.165.62:41881)] unexpected exception from downstream on [id: 0xec866440, L:/127.0.0.1:50348 - R:/127.27.165.62:41881], [8ms] delaying RPC due to: Network error: [peer master-127.27.165.62:41881(127.27.165.62:41881)] unexpected exception from downstream on [id: 0xec866440, L:/127.0.0.1:50348 - R:/127.27.165.62:41881], [28ms] refreshing cache from master, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [30ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [30ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [31ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [31ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [82ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [101ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [101ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [102ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [141ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [141ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [142ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [242ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [242ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [242ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [339ms] refreshing cache from master, [339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [341ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [341ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [342ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [342ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [819ms] refreshing cache from master, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [820ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [820ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [821ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [821ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [1459ms] refreshing cache from master, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [1460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [1461ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [1461ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [1461ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [1462ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [2840ms] refreshing cache from master, [2840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [2840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [2840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [2841ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [2842ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [2842ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [2842ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [5619ms] refreshing cache from master, [5619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [5620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [5620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [5621ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [5621ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [5622ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [5622ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [6959ms] refreshing cache from master, [6959ms] too many traces: truncated at 100 traces, deferred=Deferred@1972033969(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$123/851638263@76f46121 -> (continuation of Deferred@1337940589 after retry RPC after error@1727061712) -> (continuation of Deferred@1167377646 after retry RPC after error@272390164) -> (continuation of Deferred@627377228 after retry RPC after error@1166235758) -> (continuation of Deferred@651380612 after retry RPC after error@1423787954) -> (continuation of Deferred@1503100417 after retry RPC after error@101191029) -> (continuation of Deferred@979076439 after retry RPC after error@75867513) -> (continuation of Deferred@1719706782 after retry RPC after error@547716766) -> (continuation of Deferred@1186080480 after retry RPC after error@1874197794) -> (continuation of Deferred@217186334 after retry RPC after error@120114832) -> (continuation of Deferred@1358589363 after retry RPC after error@275175011) -> (continuation of Deferred@210290305 after retry RPC after error@1301661134) -> (continuation of Deferred@1317125690 after retry RPC after error@1760734279) -> (continuation of Deferred@579126038 after retry RPC after error@1096599869) -> (continuation of Deferred@40886082 after retry RPC after error@2139282396) -> (continuation of Deferred@2065316681 after retry RPC after error@1554513974) -> (continuation of Deferred@541147438 after retry RPC after error@1232351937) -> (continuation of Deferred@1662949589 after retry RPC after error@2133560624) -> (continuation of Deferred@1439296458 after retry RPC after error@1027815816) -> (continuation of Deferred@705484485 after retry RPC after error@166563001) -> (continuation of Deferred@325100021 after retry RPC after error@936072552) -> (continuation of Deferred@2075989930 after retry RPC after error@508609568) -> (continuation of Deferred@136950038 after retry RPC after error@1696912750) -> (continuation of Deferred@1983352359 after retry RPC after error@1851663621) -> (continuation of Deferred@725260505 after retry RPC after error@1752610677) -> (continuation of Deferred@578444010 after retry RPC after error@1024669101) -> (continuation of Deferred@1108381318 after retry RPC after error@528503982) -> (continuation of Deferred@1868427454 after retry RPC after error@1271340709) -> (continuation of Deferred@224775653 after retry RPC after error@274587075) -> (continuation of Deferred@268769142 after retry RPC after error@1665405413) -> (continuation of Deferred@1089335949 after retry RPC after error@1042324630) -> (continuation of Deferred@111579779 after retry RPC after error@1160671371) -> (continuation of Deferred@1794673465 after retry RPC after error@1350987484) -> (continuation of Deferred@1644612805 after retry RPC after error@1411007014) -> (continuation of Deferred@2048244447 after retry RPC after error@419875791) -> (continuation of Deferred@1295146190 after retry RPC after error@1262435388) -> (continuation of Deferred@472622278 after retry RPC after error@870075455) -> (continuation of Deferred@1817682598 after retry RPC after error@1589362124) -> (continuation of Deferred@536297289 after retry RPC after error@1310401852) -> (continuation of Deferred@520678603 after retry RPC after error@475952955), errback=org.apache.kudu.client.KuduTransaction$$Lambda$124/1436436397@3a751e8e -> (continuation of Deferred@1337940589 after retry RPC after error@1727061712) -> (continuation of Deferred@1167377646 after retry RPC after error@272390164) -> (continuation of Deferred@627377228 after retry RPC after error@1166235758) -> (continuation of Deferred@651380612 after retry RPC after error@1423787954) -> (continuation of Deferred@1503100417 after retry RPC after error@101191029) -> (continuation of Deferred@979076439 after retry RPC after error@75867513) -> (continuation of Deferred@1719706782 after retry RPC after error@547716766) -> (continuation of Deferred@1186080480 after retry RPC after error@1874197794) -> (continuation of Deferred@217186334 after retry RPC after error@120114832) -> (continuation of Deferred@1358589363 after retry RPC after error@275175011) -> (continuation of Deferred@210290305 after retry RPC after error@1301661134) -> (continuation of Deferred@1317125690 after retry RPC after error@1760734279) -> (continuation of Deferred@579126038 after retry RPC after error@1096599869) -> (continuation of Deferred@40886082 after retry RPC after error@2139282396) -> (continuation of Deferred@2065316681 after retry RPC after error@1554513974) -> (continuation of Deferred@541147438 after retry RPC after error@1232351937) -> (continuation of Deferred@1662949589 after retry RPC after error@2133560624) -> (continuation of Deferred@1439296458 after retry RPC after error@1027815816) -> (continuation of Deferred@705484485 after retry RPC after error@166563001) -> (continuation of Deferred@325100021 after retry RPC after error@936072552) -> (continuation of Deferred@2075989930 after retry RPC after error@508609568) -> (continuation of Deferred@136950038 after retry RPC after error@1696912750) -> (continuation of Deferred@1983352359 after retry RPC after error@1851663621) -> (continuation of Deferred@725260505 after retry RPC after error@1752610677) -> (continuation of Deferred@578444010 after retry RPC after error@1024669101) -> (continuation of Deferred@1108381318 after retry RPC after error@528503982) -> (continuation of Deferred@1868427454 after retry RPC after error@1271340709) -> (continuation of Deferred@224775653 after retry RPC after error@274587075) -> (continuation of Deferred@268769142 after retry RPC after error@1665405413) -> (continuation of Deferred@1089335949 after retry RPC after error@1042324630) -> (continuation of Deferred@111579779 after retry RPC after error@1160671371) -> (continuation of Deferred@1794673465 after retry RPC after error@1350987484) -> (continuation of Deferred@1644612805 after retry RPC after error@1411007014) -> (continuation of Deferred@2048244447 after retry RPC after error@419875791) -> (continuation of Deferred@1295146190 after retry RPC after error@1262435388) -> (continuation of Deferred@472622278 after retry RPC after error@870075455) -> (continuation of Deferred@1817682598 after retry RPC after error@1589362124) -> (continuation of Deferred@536297289 after retry RPC after error@1310401852) -> (continuation of Deferred@520678603 after retry RPC after error@475952955)))
21:01:15.822 [DEBUG - main] (Connection.java:575) [peer 22a807342d864d3b97a1a97e92484b22(127.27.165.1:45079)] disconnecting while in state READY
21:01:15.822 [DEBUG - main] (Connection.java:575) [peer master-127.27.165.62:41881(127.27.165.62:41881)] disconnecting while in state TERMINATED
21:01:15.822 [DEBUG - main] (Connection.java:575) [peer master-127.27.165.60:39093(127.27.165.60:39093)] disconnecting while in state TERMINATED
21:01:15.822 [DEBUG - main] (Connection.java:575) [peer master-127.27.165.61:46277(127.27.165.61:46277)] disconnecting while in state READY
21:01:15.823 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:01:15.824 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 22a807342d864d3b97a1a97e92484b22(127.27.165.1:45079)] handling channelInactive
21:01:15.824 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 22a807342d864d3b97a1a97e92484b22(127.27.165.1:45079)] cleaning up while in state READY due to: connection closed
21:01:15.825 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.27.165.61:46277(127.27.165.61:46277)] handling channelInactive
21:01:15.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.165.61:46277(127.27.165.61:46277)] cleaning up while in state READY due to: connection closed
21:01:15.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:15.824903 28308 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu pid 28615
21:01:15.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:15.835224 28752 generic_service.cc:196] Checking for leaks (request via RPC)
21:01:15.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:15.993279 28308 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu with pid 28615
21:01:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:16.020066 28778 connection.cc:537] client connection to 127.27.165.1:45079 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:01:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:16.020344 28778 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:01:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:16.020491 28308 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu pid 28762
21:01:16.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:16.023011 28778 consensus_peers.cc:597] T 5ff37ea97c994540928e18315264109d P 07a12c4fa3e24352a99078a970f5d452 -> Peer 22a807342d864d3b97a1a97e92484b22 (127.27.165.1:45079): Couldn't send request to peer 22a807342d864d3b97a1a97e92484b22. Status: Network error: Client connection negotiation failed: client connection to 127.27.165.1:45079: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:01:16.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:16.028370 28895 generic_service.cc:196] Checking for leaks (request via RPC)
21:01:16.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:16.141279 28308 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu with pid 28762
21:01:16.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251209 21:01:16.162000 28417 connection.cc:537] server connection from 127.27.165.2:45295 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:01:16.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:16.162232 28308 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu pid 28905
21:01:16.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:16.169659 29037 generic_service.cc:196] Checking for leaks (request via RPC)
21:01:16.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:16.279352 28308 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu with pid 28905
21:01:16.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:16.299700 28308 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu pid 28397
21:01:16.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:16.300990 28471 generic_service.cc:196] Checking for leaks (request via RPC)
21:01:16.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251209 21:01:16.433360 28308 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV3TXV2/build/asan/bin/kudu with pid 28397
21:01:16.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-09T21:01:16Z chronyd exiting
21:01:16.602 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=Kudu Master, attempt=41, TimeoutTracker(timeout=50000, elapsed=47342), Traces: [0ms] sending RPC to server master-127.27.165.62:41881, [7ms] received response from server master-127.27.165.62:41881: Network error: [peer master-127.27.165.62:41881(127.27.165.62:41881)] unexpected exception from downstream on [id: 0xec866440, L:/127.0.0.1:50348 - R:/127.27.165.62:41881], [8ms] delaying RPC due to: Network error: [peer master-127.27.165.62:41881(127.27.165.62:41881)] unexpected exception from downstream on [id: 0xec866440, L:/127.0.0.1:50348 - R:/127.27.165.62:41881], [28ms] refreshing cache from master, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [30ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [30ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [31ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [31ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [82ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [101ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [101ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [102ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [141ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [141ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [142ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [242ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [242ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [242ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [339ms] refreshing cache from master, [339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [341ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [341ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [342ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [342ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [819ms] refreshing cache from master, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [820ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [820ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [821ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [821ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [1459ms] refreshing cache from master, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [1460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [1461ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [1461ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [1461ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [1462ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [2840ms] refreshing cache from master, [2840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [2840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [2840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [2841ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [2842ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [2842ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [2842ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [5619ms] refreshing cache from master, [5619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [5620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [5620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [5621ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [5621ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [5622ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [5622ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [6959ms] refreshing cache from master, [6959ms] too many traces: truncated at 100 traces, deferred=Deferred@1972033969(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$123/851638263@76f46121 -> (continuation of Deferred@1337940589 after retry RPC after error@1727061712) -> (continuation of Deferred@1167377646 after retry RPC after error@272390164) -> (continuation of Deferred@627377228 after retry RPC after error@1166235758) -> (continuation of Deferred@651380612 after retry RPC after error@1423787954) -> (continuation of Deferred@1503100417 after retry RPC after error@101191029) -> (continuation of Deferred@979076439 after retry RPC after error@75867513) -> (continuation of Deferred@1719706782 after retry RPC after error@547716766) -> (continuation of Deferred@1186080480 after retry RPC after error@1874197794) -> (continuation of Deferred@217186334 after retry RPC after error@120114832) -> (continuation of Deferred@1358589363 after retry RPC after error@275175011) -> (continuation of Deferred@210290305 after retry RPC after error@1301661134) -> (continuation of Deferred@1317125690 after retry RPC after error@1760734279) -> (continuation of Deferred@579126038 after retry RPC after error@1096599869) -> (continuation of Deferred@40886082 after retry RPC after error@2139282396) -> (continuation of Deferred@2065316681 after retry RPC after error@1554513974) -> (continuation of Deferred@541147438 after retry RPC after error@1232351937) -> (continuation of Deferred@1662949589 after retry RPC after error@2133560624) -> (continuation of Deferred@1439296458 after retry RPC after error@1027815816) -> (continuation of Deferred@705484485 after retry RPC after error@166563001) -> (continuation of Deferred@325100021 after retry RPC after error@936072552) -> (continuation of Deferred@2075989930 after retry RPC after error@508609568) -> (continuation of Deferred@136950038 after retry RPC after error@1696912750) -> (continuation of Deferred@1983352359 after retry RPC after error@1851663621) -> (continuation of Deferred@725260505 after retry RPC after error@1752610677) -> (continuation of Deferred@578444010 after retry RPC after error@1024669101) -> (continuation of Deferred@1108381318 after retry RPC after error@528503982) -> (continuation of Deferred@1868427454 after retry RPC after error@1271340709) -> (continuation of Deferred@224775653 after retry RPC after error@274587075) -> (continuation of Deferred@268769142 after retry RPC after error@1665405413) -> (continuation of Deferred@1089335949 after retry RPC after error@1042324630) -> (continuation of Deferred@111579779 after retry RPC after error@1160671371) -> (continuation of Deferred@1794673465 after retry RPC after error@1350987484) -> (continuation of Deferred@1644612805 after retry RPC after error@1411007014) -> (continuation of Deferred@2048244447 after retry RPC after error@419875791) -> (continuation of Deferred@1295146190 after retry RPC after error@1262435388) -> (continuation of Deferred@472622278 after retry RPC after error@870075455) -> (continuation of Deferred@1817682598 after retry RPC after error@1589362124) -> (continuation of Deferred@536297289 after retry RPC after error@1310401852) -> (continuation of Deferred@520678603 after retry RPC after error@475952955), errback=org.apache.kudu.client.KuduTransaction$$Lambda$124/1436436397@3a751e8e -> (continuation of Deferred@1337940589 after retry RPC after error@1727061712) -> (continuation of Deferred@1167377646 after retry RPC after error@272390164) -> (continuation of Deferred@627377228 after retry RPC after error@1166235758) -> (continuation of Deferred@651380612 after retry RPC after error@1423787954) -> (continuation of Deferred@1503100417 after retry RPC after error@101191029) -> (continuation of Deferred@979076439 after retry RPC after error@75867513) -> (continuation of Deferred@1719706782 after retry RPC after error@547716766) -> (continuation of Deferred@1186080480 after retry RPC after error@1874197794) -> (continuation of Deferred@217186334 after retry RPC after error@120114832) -> (continuation of Deferred@1358589363 after retry RPC after error@275175011) -> (continuation of Deferred@210290305 after retry RPC after error@1301661134) -> (continuation of Deferred@1317125690 after retry RPC after error@1760734279) -> (continuation of Deferred@579126038 after retry RPC after error@1096599869) -> (continuation of Deferred@40886082 after retry RPC after error@2139282396) -> (continuation of Deferred@2065316681 after retry RPC after error@1554513974) -> (continuation of Deferred@541147438 after retry RPC after error@1232351937) -> (continuation of Deferred@1662949589 after retry RPC after error@2133560624) -> (continuation of Deferred@1439296458 after retry RPC after error@1027815816) -> (continuation of Deferred@705484485 after retry RPC after error@166563001) -> (continuation of Deferred@325100021 after retry RPC after error@936072552) -> (continuation of Deferred@2075989930 after retry RPC after error@508609568) -> (continuation of Deferred@136950038 after retry RPC after error@1696912750) -> (continuation of Deferred@1983352359 after retry RPC after error@1851663621) -> (continuation of Deferred@725260505 after retry RPC after error@1752610677) -> (continuation of Deferred@578444010 after retry RPC after error@1024669101) -> (continuation of Deferred@1108381318 after retry RPC after error@528503982) -> (continuation of Deferred@1868427454 after retry RPC after error@1271340709) -> (continuation of Deferred@224775653 after retry RPC after error@274587075) -> (continuation of Deferred@268769142 after retry RPC after error@1665405413) -> (continuation of Deferred@1089335949 after retry RPC after error@1042324630) -> (continuation of Deferred@111579779 after retry RPC after error@1160671371) -> (continuation of Deferred@1794673465 after retry RPC after error@1350987484) -> (continuation of Deferred@1644612805 after retry RPC after error@1411007014) -> (continuation of Deferred@2048244447 after retry RPC after error@419875791) -> (continuation of Deferred@1295146190 after retry RPC after error@1262435388) -> (continuation of Deferred@472622278 after retry RPC after error@870075455) -> (continuation of Deferred@1817682598 after retry RPC after error@1589362124) -> (continuation of Deferred@536297289 after retry RPC after error@1310401852) -> (continuation of Deferred@520678603 after retry RPC after error@475952955)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:553) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.165.62:41881
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=Kudu Master, attempt=41, TimeoutTracker(timeout=50000, elapsed=47342), Traces: [0ms] sending RPC to server master-127.27.165.62:41881, [7ms] received response from server master-127.27.165.62:41881: Network error: [peer master-127.27.165.62:41881(127.27.165.62:41881)] unexpected exception from downstream on [id: 0xec866440, L:/127.0.0.1:50348 - R:/127.27.165.62:41881], [8ms] delaying RPC due to: Network error: [peer master-127.27.165.62:41881(127.27.165.62:41881)] unexpected exception from downstream on [id: 0xec866440, L:/127.0.0.1:50348 - R:/127.27.165.62:41881], [28ms] refreshing cache from master, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [30ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [30ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [31ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [31ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [41ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [82ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [101ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [101ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [102ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [141ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [141ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [142ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [242ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [242ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [242ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [339ms] refreshing cache from master, [339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [341ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [341ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [342ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [342ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [819ms] refreshing cache from master, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [820ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [820ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [821ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [821ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [1459ms] refreshing cache from master, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [1460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [1461ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [1461ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [1461ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [1462ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [2840ms] refreshing cache from master, [2840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [2840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [2840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [2841ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [2842ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [2842ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [2842ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [5619ms] refreshing cache from master, [5619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.60:39093, [5620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.61:46277, [5620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.165.62:41881, [5621ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.60:39093: Network error: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093, [5621ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.62:41881: Network error: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [5622ms] Sub RPC ConnectToMaster: received response from server master-127.27.165.61:46277: OK, [5622ms] delaying RPC due to: Service unavailable: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881, [6959ms] refreshing cache from master, [6959ms] too many traces: truncated at 100 traces, deferred=Deferred@1972033969(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$123/851638263@76f46121 -> (continuation of Deferred@1337940589 after retry RPC after error@1727061712) -> (continuation of Deferred@1167377646 after retry RPC after error@272390164) -> (continuation of Deferred@627377228 after retry RPC after error@1166235758) -> (continuation of Deferred@651380612 after retry RPC after error@1423787954) -> (continuation of Deferred@1503100417 after retry RPC after error@101191029) -> (continuation of Deferred@979076439 after retry RPC after error@75867513) -> (continuation of Deferred@1719706782 after retry RPC after error@547716766) -> (continuation of Deferred@1186080480 after retry RPC after error@1874197794) -> (continuation of Deferred@217186334 after retry RPC after error@120114832) -> (continuation of Deferred@1358589363 after retry RPC after error@275175011) -> (continuation of Deferred@210290305 after retry RPC after error@1301661134) -> (continuation of Deferred@1317125690 after retry RPC after error@1760734279) -> (continuation of Deferred@579126038 after retry RPC after error@1096599869) -> (continuation of Deferred@40886082 after retry RPC after error@2139282396) -> (continuation of Deferred@2065316681 after retry RPC after error@1554513974) -> (continuation of Deferred@541147438 after retry RPC after error@1232351937) -> (continuation of Deferred@1662949589 after retry RPC after error@2133560624) -> (continuation of Deferred@1439296458 after retry RPC after error@1027815816) -> (continuation of Deferred@705484485 after retry RPC after error@166563001) -> (continuation of Deferred@325100021 after retry RPC after error@936072552) -> (continuation of Deferred@2075989930 after retry RPC after error@508609568) -> (continuation of Deferred@136950038 after retry RPC after error@1696912750) -> (continuation of Deferred@1983352359 after retry RPC after error@1851663621) -> (continuation of Deferred@725260505 after retry RPC after error@1752610677) -> (continuation of Deferred@578444010 after retry RPC after error@1024669101) -> (continuation of Deferred@1108381318 after retry RPC after error@528503982) -> (continuation of Deferred@1868427454 after retry RPC after error@1271340709) -> (continuation of Deferred@224775653 after retry RPC after error@274587075) -> (continuation of Deferred@268769142 after retry RPC after error@1665405413) -> (continuation of Deferred@1089335949 after retry RPC after error@1042324630) -> (continuation of Deferred@111579779 after retry RPC after error@1160671371) -> (continuation of Deferred@1794673465 after retry RPC after error@1350987484) -> (continuation of Deferred@1644612805 after retry RPC after error@1411007014) -> (continuation of Deferred@2048244447 after retry RPC after error@419875791) -> (continuation of Deferred@1295146190 after retry RPC after error@1262435388) -> (continuation of Deferred@472622278 after retry RPC after error@870075455) -> (continuation of Deferred@1817682598 after retry RPC after error@1589362124) -> (continuation of Deferred@536297289 after retry RPC after error@1310401852) -> (continuation of Deferred@520678603 after retry RPC after error@475952955), errback=org.apache.kudu.client.KuduTransaction$$Lambda$124/1436436397@3a751e8e -> (continuation of Deferred@1337940589 after retry RPC after error@1727061712) -> (continuation of Deferred@1167377646 after retry RPC after error@272390164) -> (continuation of Deferred@627377228 after retry RPC after error@1166235758) -> (continuation of Deferred@651380612 after retry RPC after error@1423787954) -> (continuation of Deferred@1503100417 after retry RPC after error@101191029) -> (continuation of Deferred@979076439 after retry RPC after error@75867513) -> (continuation of Deferred@1719706782 after retry RPC after error@547716766) -> (continuation of Deferred@1186080480 after retry RPC after error@1874197794) -> (continuation of Deferred@217186334 after retry RPC after error@120114832) -> (continuation of Deferred@1358589363 after retry RPC after error@275175011) -> (continuation of Deferred@210290305 after retry RPC after error@1301661134) -> (continuation of Deferred@1317125690 after retry RPC after error@1760734279) -> (continuation of Deferred@579126038 after retry RPC after error@1096599869) -> (continuation of Deferred@40886082 after retry RPC after error@2139282396) -> (continuation of Deferred@2065316681 after retry RPC after error@1554513974) -> (continuation of Deferred@541147438 after retry RPC after error@1232351937) -> (continuation of Deferred@1662949589 after retry RPC after error@2133560624) -> (continuation of Deferred@1439296458 after retry RPC after error@1027815816) -> (continuation of Deferred@705484485 after retry RPC after error@166563001) -> (continuation of Deferred@325100021 after retry RPC after error@936072552) -> (continuation of Deferred@2075989930 after retry RPC after error@508609568) -> (continuation of Deferred@136950038 after retry RPC after error@1696912750) -> (continuation of Deferred@1983352359 after retry RPC after error@1851663621) -> (continuation of Deferred@725260505 after retry RPC after error@1752610677) -> (continuation of Deferred@578444010 after retry RPC after error@1024669101) -> (continuation of Deferred@1108381318 after retry RPC after error@528503982) -> (continuation of Deferred@1868427454 after retry RPC after error@1271340709) -> (continuation of Deferred@224775653 after retry RPC after error@274587075) -> (continuation of Deferred@268769142 after retry RPC after error@1665405413) -> (continuation of Deferred@1089335949 after retry RPC after error@1042324630) -> (continuation of Deferred@111579779 after retry RPC after error@1160671371) -> (continuation of Deferred@1794673465 after retry RPC after error@1350987484) -> (continuation of Deferred@1644612805 after retry RPC after error@1411007014) -> (continuation of Deferred@2048244447 after retry RPC after error@419875791) -> (continuation of Deferred@1295146190 after retry RPC after error@1262435388) -> (continuation of Deferred@472622278 after retry RPC after error@870075455) -> (continuation of Deferred@1817682598 after retry RPC after error@1589362124) -> (continuation of Deferred@536297289 after retry RPC after error@1310401852) -> (continuation of Deferred@520678603 after retry RPC after error@475952955)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:553)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.27.165.60:39093,127.27.165.61:46277,127.27.165.62:41881) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.60:39093(127.27.165.60:39093): Connection refused: /127.27.165.60:39093,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.165.62:41881(127.27.165.62:41881): Connection refused: /127.27.165.62:41881
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.165.62:41881
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)