19:59:20.032 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
19:59:20.032 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task3Meco8/test-tmp
19:59:20.032 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
19:59:20.032 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task3Meco8/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
19:59:20.033 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
19:59:20.336 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:59:20.336 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:59:20.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.338928 8610 mini_oidc.cc:150] Starting JWKS server
19:59:20.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.341357 8610 webserver.cc:480] Webserver started at http://127.0.0.1:36827/ using document root <none> and password file <none>
19:59:20.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.341588 8610 mini_oidc.cc:186] Starting OIDC Discovery server
19:59:20.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.342082 8610 webserver.cc:480] Webserver started at http://127.0.0.1:40797/ using document root <none> and password file <none>
19:59:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T19:59:20Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
19:59:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T19:59:20Z Disabled control of system clock
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.369972 8610 external_mini_cluster.cc:1366] Running /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/logs
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data/info.pb
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.190:42679
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.190
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:59:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:59:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36827/jwks.json
19:59:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
19:59:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
19:59:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.664158 8621 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:59:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.664436 8621 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.664516 8621 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:59:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.671566 8621 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:59:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.671665 8621 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
19:59:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.671711 8621 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:59:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.671749 8621 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:59:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.671800 8621 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:59:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.671837 8621 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:59:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.671885 8621 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36827/jwks.json
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.680203 8621 master_runner.cc:386] Master server non-default flags:
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.190:42679
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36827/jwks.json
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data/info.pb
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.190
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/logs
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:20.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
19:59:20.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:20.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 19:43:25 UTC on 24a791456cd2
19:59:20.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7836
19:59:20.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:20.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.681648 8621 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:20.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.683274 8621 file_cache.cc:492] Constructed file cache file cache with capacity 419430
19:59:20.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.690616 8627 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:20.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.690563 8626 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:20.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.691543 8629 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:20.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.691589 8621 server_base.cc:1047] running on GCE node
19:59:20.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.692303 8621 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:20.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.693559 8621 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:20.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.694797 8621 hybrid_clock.cc:648] HybridClock initialized: now 1756670360694718 us; error 37 us; skew 500 ppm
19:59:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.695108 8621 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:20.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.697140 8621 webserver.cc:480] Webserver started at http://127.8.104.190:46569/ using document root <none> and password file <none>
19:59:20.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.697847 8621 fs_manager.cc:362] Metadata directory not provided
19:59:20.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.697970 8621 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:20.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.698251 8621 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
19:59:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.700369 8621 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data/instance:
19:59:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e11e01529b2f4bf88c980fdc7cc21f92"
19:59:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:20 on dist-test-slave-7hw0"
19:59:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.701087 8621 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal/instance:
19:59:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e11e01529b2f4bf88c980fdc7cc21f92"
19:59:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:20 on dist-test-slave-7hw0"
19:59:20.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.705763 8621 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.008s sys 0.000s
19:59:20.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.708781 8635 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.709975 8621 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.710145 8621 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data,/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e11e01529b2f4bf88c980fdc7cc21f92"
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:20 on dist-test-slave-7hw0"
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.710312 8621 fs_report.cc:389] FS layout report
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data/data
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:20.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.735626 8621 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
19:59:20.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.736526 8621 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:20.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.736841 8621 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:20.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.759747 8621 rpc_server.cc:307] RPC server started. Bound to: 127.8.104.190:42679
19:59:20.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.759786 8696 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.104.190:42679 every 8 connection(s)
19:59:20.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.761296 8621 server_base.cc:1179] Dumped server information to /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data/info.pb
19:59:20.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.765225 8697 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:20.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.766312 8610 external_mini_cluster.cc:1428] Started /tmp/dist-test-task3Meco8/build/asan/bin/kudu as pid 8621
19:59:20.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.766624 8610 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal/instance
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.769480 8610 external_mini_cluster.cc:1366] Running /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/wal
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/data
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/logs
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/data/info.pb
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.189:34439
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.189
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36827/jwks.json
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
19:59:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
19:59:20.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.773851 8697 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } has no permanent_uuid. Determining permanent_uuid...
19:59:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.795599 8697 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } has no permanent_uuid. Determining permanent_uuid...
19:59:20.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.796501 8637 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:20.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.797878 8697 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.847147 8697 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } attempt: 1
19:59:20.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.848865 8697 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:20.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:20.917091 8697 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } attempt: 2
19:59:20.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:20.918692 8697 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:21.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.009920 8697 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } attempt: 3
19:59:21.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.011610 8697 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.070964 8699 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:59:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.071286 8699 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.071364 8699 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:59:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.078979 8699 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:59:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.079077 8699 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
19:59:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.079121 8699 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:59:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.079159 8699 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:59:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.079207 8699 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:59:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.079243 8699 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:59:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.079282 8699 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36827/jwks.json
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.088135 8699 master_runner.cc:386] Master server non-default flags:
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/data
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/wal
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.189:34439
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36827/jwks.json
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:21.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/data/info.pb
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.189
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/logs
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 19:43:25 UTC on 24a791456cd2
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7836
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.089841 8699 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:21.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.091434 8699 file_cache.cc:492] Constructed file cache file cache with capacity 419430
19:59:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.099016 8710 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.099004 8708 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.099385 8707 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.100128 8699 server_base.cc:1047] running on GCE node
19:59:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.100768 8699 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:21.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.102041 8699 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.103258 8699 hybrid_clock.cc:648] HybridClock initialized: now 1756670361103197 us; error 38 us; skew 500 ppm
19:59:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.103577 8699 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:21.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.105615 8699 webserver.cc:480] Webserver started at http://127.8.104.189:33457/ using document root <none> and password file <none>
19:59:21.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.106294 8699 fs_manager.cc:362] Metadata directory not provided
19:59:21.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.106415 8699 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:21.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.106720 8699 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
19:59:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.108949 8699 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/data/instance:
19:59:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "89ecc82c1296466abc4576789e40284d"
19:59:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:21 on dist-test-slave-7hw0"
19:59:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.109701 8699 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/wal/instance:
19:59:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "89ecc82c1296466abc4576789e40284d"
19:59:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:21 on dist-test-slave-7hw0"
19:59:21.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.114471 8699 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.003s sys 0.001s
19:59:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.117609 8716 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.118883 8699 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.119068 8699 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/data,/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/wal
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "89ecc82c1296466abc4576789e40284d"
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:21 on dist-test-slave-7hw0"
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.119270 8699 fs_report.cc:389] FS layout report
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/wal
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/wal
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/data/data
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.135742 8699 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
19:59:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.136557 8699 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.136840 8699 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:21.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.154853 8697 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } attempt: 4
19:59:21.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.156919 8697 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:21.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.158422 8699 rpc_server.cc:307] RPC server started. Bound to: 127.8.104.189:34439
19:59:21.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.158440 8777 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.104.189:34439 every 8 connection(s)
19:59:21.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.159961 8699 server_base.cc:1179] Dumped server information to /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/data/info.pb
19:59:21.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.163625 8778 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.168433 8610 external_mini_cluster.cc:1428] Started /tmp/dist-test-task3Meco8/build/asan/bin/kudu as pid 8699
19:59:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.168562 8610 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-1/wal/instance
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.169023 8778 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } has no permanent_uuid. Determining permanent_uuid...
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.169433 8610 external_mini_cluster.cc:1366] Running /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/wal
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/data
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/logs
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/data/info.pb
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.188:43579
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.188
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36827/jwks.json
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
19:59:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
19:59:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.181953 8778 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } has no permanent_uuid. Determining permanent_uuid...
19:59:21.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.190331 8778 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } has no permanent_uuid. Determining permanent_uuid...
19:59:21.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.191416 8720 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:21.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.192911 8778 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.188:43579: Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.242203 8778 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } attempt: 1
19:59:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.243917 8778 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.188:43579: Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:21.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.312116 8778 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } attempt: 2
19:59:21.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.314066 8778 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.188:43579: Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:21.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.405382 8778 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } attempt: 3
19:59:21.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.407382 8778 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.188:43579: Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:21.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.456195 8697 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } attempt: 5
19:59:21.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.462353 8697 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } has no permanent_uuid. Determining permanent_uuid...
19:59:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.464097 8697 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.188:43579: Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:21.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.478277 8781 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:59:21.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.478603 8781 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:21.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.478731 8781 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:59:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.486116 8781 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:59:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.486224 8781 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
19:59:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.486289 8781 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:59:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.486343 8781 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:59:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.486415 8781 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:59:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.486469 8781 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:59:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.486541 8781 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36827/jwks.json
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.495378 8781 master_runner.cc:386] Master server non-default flags:
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/data
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/wal
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:59:21.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.188:43579
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36827/jwks.json
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/data/info.pb
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.188
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/logs
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 19:43:25 UTC on 24a791456cd2
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7836
19:59:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.497252 8781 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:21.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.498976 8781 file_cache.cc:492] Constructed file cache file cache with capacity 419430
19:59:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.506057 8788 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.506114 8789 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:21.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.510504 8791 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:21.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.510805 8781 server_base.cc:1047] running on GCE node
19:59:21.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.511519 8781 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:21.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.512789 8781 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:21.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.514010 8781 hybrid_clock.cc:648] HybridClock initialized: now 1756670361513951 us; error 43 us; skew 500 ppm
19:59:21.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.514312 8781 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:21.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.515291 8697 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } attempt: 1
19:59:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.516575 8781 webserver.cc:480] Webserver started at http://127.8.104.188:38037/ using document root <none> and password file <none>
19:59:21.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.517091 8697 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.188:43579: Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:21.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.517367 8781 fs_manager.cc:362] Metadata directory not provided
19:59:21.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.517468 8781 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:21.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.517796 8781 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
19:59:21.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.520023 8781 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/data/instance:
19:59:21.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa8c3788cc4a40c5bb7b2497e6bab159"
19:59:21.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:21 on dist-test-slave-7hw0"
19:59:21.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.520710 8781 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/wal/instance:
19:59:21.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa8c3788cc4a40c5bb7b2497e6bab159"
19:59:21.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:21 on dist-test-slave-7hw0"
19:59:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.525674 8781 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.002s
19:59:21.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.528887 8797 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.530174 8781 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.530310 8781 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/data,/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/wal
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa8c3788cc4a40c5bb7b2497e6bab159"
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:21 on dist-test-slave-7hw0"
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.530447 8781 fs_report.cc:389] FS layout report
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/wal
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/wal
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/data/data
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.550601 8778 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } attempt: 4
19:59:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.552309 8778 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.8.104.188:43579: Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.561954 8781 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
19:59:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.562701 8781 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:21.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.562933 8781 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.584026 8781 rpc_server.cc:307] RPC server started. Bound to: 127.8.104.188:43579
19:59:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.584046 8858 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.104.188:43579 every 8 connection(s)
19:59:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.585281 8697 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } attempt: 2
19:59:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.585386 8781 server_base.cc:1179] Dumped server information to /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/data/info.pb
19:59:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.587327 8610 external_mini_cluster.cc:1428] Started /tmp/dist-test-task3Meco8/build/asan/bin/kudu as pid 8781
19:59:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.587548 8610 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-2/wal/instance
19:59:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.589968 8859 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:21.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.598579 8859 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } has no permanent_uuid. Determining permanent_uuid...
19:59:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.605801 8697 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: Bootstrap starting.
19:59:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.609839 8697 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: Neither blocks nor log segments found. Creating new log.
19:59:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.610054 8859 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } has no permanent_uuid. Determining permanent_uuid...
19:59:21.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.611009 8697 log.cc:826] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: Log is configured to *not* fsync() on all Append() calls
19:59:21.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.617466 8697 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: No bootstrap required, opened a new log
19:59:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.622591 8859 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } has no permanent_uuid. Determining permanent_uuid...
19:59:21.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.624967 8697 raft_consensus.cc:357] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:21.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.625245 8697 raft_consensus.cc:383] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:21.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.625340 8697 raft_consensus.cc:738] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e11e01529b2f4bf88c980fdc7cc21f92, State: Initialized, Role: FOLLOWER
19:59:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.625995 8697 consensus_queue.cc:260] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:21.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.626871 8868 sys_catalog.cc:455] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:21.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.627638 8868 sys_catalog.cc:458] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: This master's current role is: FOLLOWER
19:59:21.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.628005 8697 sys_catalog.cc:564] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: configured and running, proceeding with master startup.
19:59:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.631369 8859 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159: Bootstrap starting.
19:59:21.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.634775 8859 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159: Neither blocks nor log segments found. Creating new log.
19:59:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.636049 8859 log.cc:826] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159: Log is configured to *not* fsync() on all Append() calls
19:59:21.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.637859 8879 catalog_manager.cc:1560] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:59:21.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.637987 8879 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:59:21.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.640087 8868 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:59:21.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.640316 8868 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:21.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.641856 8868 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:21.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.637346 8621 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: OK, instance_id { permanent_uuid: "89ecc82c1296466abc4576789e40284d" instance_seqno: 1756670361152960 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:59:21.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.648293 8621 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:59:21.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.651151 8859 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159: No bootstrap required, opened a new log
19:59:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.657969 8859 raft_consensus.cc:357] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.658344 8859 raft_consensus.cc:383] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.658452 8859 raft_consensus.cc:738] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa8c3788cc4a40c5bb7b2497e6bab159, State: Initialized, Role: FOLLOWER
19:59:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.659114 8859 consensus_queue.cc:260] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:21.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.660012 8882 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:21.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.660266 8882 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: This master's current role is: FOLLOWER
19:59:21.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.661145 8859 sys_catalog.cc:564] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: configured and running, proceeding with master startup.
19:59:21.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.661202 8824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" is_pre_election: true
19:59:21.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.661749 8824 raft_consensus.cc:2466] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e11e01529b2f4bf88c980fdc7cc21f92 in term 0.
19:59:21.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.662420 8639 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92, fa8c3788cc4a40c5bb7b2497e6bab159; no voters:
19:59:21.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.662958 8868 raft_consensus.cc:2802] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 0 FOLLOWER]: Leader pre-election won for term 1
19:59:21.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.663106 8868 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:59:21.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.663205 8868 raft_consensus.cc:3058] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 0 FOLLOWER]: Advancing to term 1
19:59:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.666266 8868 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.667527 8824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159"
19:59:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.667806 8824 raft_consensus.cc:3058] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 0 FOLLOWER]: Advancing to term 1
19:59:21.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.669109 8868 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 1 election: Requested vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.670974 8824 raft_consensus.cc:2466] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e11e01529b2f4bf88c980fdc7cc21f92 in term 1.
19:59:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.671341 8639 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92, fa8c3788cc4a40c5bb7b2497e6bab159; no voters:
19:59:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.671733 8868 raft_consensus.cc:2802] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 1 FOLLOWER]: Leader election won for term 1
19:59:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.679963 8868 raft_consensus.cc:695] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 1 LEADER]: Becoming Leader. State: Replica: e11e01529b2f4bf88c980fdc7cc21f92, State: Running, Role: LEADER
19:59:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.680532 8868 consensus_queue.cc:237] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.688884 8887 sys_catalog.cc:455] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e11e01529b2f4bf88c980fdc7cc21f92. Latest consensus state: current_term: 1 leader_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.689455 8887 sys_catalog.cc:458] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: This master's current role is: LEADER
19:59:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.702212 8893 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
19:59:21.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.705248 8893 catalog_manager.cc:1486] Initializing Kudu cluster ID...
19:59:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.712584 8781 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: OK, instance_id { permanent_uuid: "89ecc82c1296466abc4576789e40284d" instance_seqno: 1756670361152960 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:59:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.722098 8781 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:59:21.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.723187 8896 catalog_manager.cc:1560] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:59:21.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.723595 8896 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:59:21.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.735442 8824 raft_consensus.cc:1273] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 1 FOLLOWER]: Refusing update from remote peer e11e01529b2f4bf88c980fdc7cc21f92: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:59:21.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.736371 8868 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:59:21.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:21.739751 8641 tablet.cc:2378] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:59:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.746348 8882 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e11e01529b2f4bf88c980fdc7cc21f92. Latest consensus state: current_term: 1 leader_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.746649 8882 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: This master's current role is: FOLLOWER
19:59:21.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.759768 8882 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:21.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.759958 8882 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: This master's current role is: FOLLOWER
19:59:21.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.762400 8887 sys_catalog.cc:455] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:21.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.762880 8887 sys_catalog.cc:458] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: This master's current role is: LEADER
19:59:21.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.767805 8893 catalog_manager.cc:1349] Generated new cluster ID: a1178ae452f34170ababc7c815199e6f
19:59:21.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.767912 8893 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
19:59:21.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.805313 8893 catalog_manager.cc:1372] Generated new certificate authority record
19:59:21.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.808230 8893 catalog_manager.cc:1506] Loading token signing keys...
19:59:21.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.825171 8893 catalog_manager.cc:5955] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: Generated new TSK 0
19:59:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.826299 8893 catalog_manager.cc:1516] Initializing in-progress tserver states...
19:59:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.840415 8610 external_mini_cluster.cc:1366] Running /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/wal
19:59:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/data
19:59:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/logs
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/data/info.pb
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.129:0
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.104.129
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.129
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
19:59:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
19:59:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.851524 8778 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } attempt: 5
19:59:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.864343 8778 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d: Bootstrap starting.
19:59:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.868317 8778 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d: Neither blocks nor log segments found. Creating new log.
19:59:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.870844 8778 log.cc:826] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d: Log is configured to *not* fsync() on all Append() calls
19:59:21.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.885530 8778 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d: No bootstrap required, opened a new log
19:59:21.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.897455 8778 raft_consensus.cc:357] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:21.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.898525 8778 raft_consensus.cc:383] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:21.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.898936 8778 raft_consensus.cc:738] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 89ecc82c1296466abc4576789e40284d, State: Initialized, Role: FOLLOWER
19:59:21.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.900544 8778 consensus_queue.cc:260] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:21.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.913837 8743 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "89ecc82c1296466abc4576789e40284d" is_pre_election: true
19:59:21.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.914363 8743 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e11e01529b2f4bf88c980fdc7cc21f92 in term 0.
19:59:21.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.915634 8742 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "89ecc82c1296466abc4576789e40284d"
19:59:21.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.915863 8742 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 0 FOLLOWER]: Advancing to term 1
19:59:21.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.919149 8742 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e11e01529b2f4bf88c980fdc7cc21f92 in term 1.
19:59:21.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.921679 8741 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 1 FOLLOWER]: Refusing update from remote peer e11e01529b2f4bf88c980fdc7cc21f92: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:59:21.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.923393 8778 sys_catalog.cc:564] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [sys.catalog]: configured and running, proceeding with master startup.
19:59:21.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.923346 8887 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:59:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.973655 8905 sys_catalog.cc:455] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.974025 8905 sys_catalog.cc:458] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [sys.catalog]: This master's current role is: FOLLOWER
19:59:21.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.975916 8906 sys_catalog.cc:455] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [sys.catalog]: SysCatalogTable state changed. Reason: New leader e11e01529b2f4bf88c980fdc7cc21f92. Latest consensus state: current_term: 1 leader_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:21.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.976212 8906 sys_catalog.cc:458] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [sys.catalog]: This master's current role is: FOLLOWER
19:59:21.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.976742 8912 sys_catalog.cc:455] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:21.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:21.977157 8912 sys_catalog.cc:458] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [sys.catalog]: This master's current role is: FOLLOWER
19:59:22.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.018023 8922 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
19:59:22.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.032528 8887 sys_catalog.cc:455] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:22.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.032829 8887 sys_catalog.cc:458] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: This master's current role is: LEADER
19:59:22.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.264081 8903 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:59:22.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.264376 8903 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:22.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.264503 8903 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:59:22.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.271711 8903 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:59:22.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.271883 8903 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
19:59:22.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.271965 8903 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
19:59:22.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.272020 8903 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
19:59:22.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.272152 8903 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.104.129
19:59:22.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.280697 8903 tablet_server_runner.cc:78] Tablet server non-default flags:
19:59:22.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/data
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/wal
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.129:0
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/data/info.pb
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.129
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.104.129
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/logs
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
19:59:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 19:43:25 UTC on 24a791456cd2
19:59:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7836
19:59:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.282545 8903 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.284157 8903 file_cache.cc:492] Constructed file cache file cache with capacity 419430
19:59:22.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.292626 8928 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:22.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.292701 8929 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:22.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.293430 8931 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:22.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.293897 8903 server_base.cc:1047] running on GCE node
19:59:22.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.294443 8903 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.295269 8903 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.296471 8903 hybrid_clock.cc:648] HybridClock initialized: now 1756670362296414 us; error 35 us; skew 500 ppm
19:59:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.296739 8903 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.298688 8903 webserver.cc:480] Webserver started at http://127.8.104.129:39951/ using document root <none> and password file <none>
19:59:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.299355 8903 fs_manager.cc:362] Metadata directory not provided
19:59:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.299486 8903 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.299829 8903 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
19:59:22.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.302080 8903 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/data/instance:
19:59:22.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60624d7b4dcf468da997ac3454bb427a"
19:59:22.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:22 on dist-test-slave-7hw0"
19:59:22.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.302805 8903 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/wal/instance:
19:59:22.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60624d7b4dcf468da997ac3454bb427a"
19:59:22.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:22 on dist-test-slave-7hw0"
19:59:22.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.307387 8903 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.004s sys 0.001s
19:59:22.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.310302 8937 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:59:22.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.311592 8903 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
19:59:22.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.311756 8903 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/data,/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/wal
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60624d7b4dcf468da997ac3454bb427a"
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:22 on dist-test-slave-7hw0"
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.311924 8903 fs_report.cc:389] FS layout report
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/wal
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/wal
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/data/data
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:22.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.331079 8903 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
19:59:22.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.332028 8903 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:22.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.332393 8903 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:22.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.335433 8903 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:59:22.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.335683 8903 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
19:59:22.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.335973 8903 ts_tablet_manager.cc:610] Registered 0 tablets
19:59:22.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.336103 8903 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:59:22.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.407483 8903 rpc_server.cc:307] RPC server started. Bound to: 127.8.104.129:42643
19:59:22.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.407516 9060 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.104.129:42643 every 8 connection(s)
19:59:22.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.408952 8903 server_base.cc:1179] Dumped server information to /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/data/info.pb
19:59:22.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.412292 9061 heartbeater.cc:344] Connected to a master server at 127.8.104.190:42679
19:59:22.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.413286 9061 heartbeater.cc:461] Registering TS with master...
19:59:22.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.414343 9061 heartbeater.cc:507] Master 127.8.104.190:42679 requested a full tablet report, sending...
19:59:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.416657 8652 ts_manager.cc:194] Registered new tserver with Master: 60624d7b4dcf468da997ac3454bb427a (127.8.104.129:42643)
19:59:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.417557 9063 heartbeater.cc:344] Connected to a master server at 127.8.104.189:34439
19:59:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.417702 9063 heartbeater.cc:461] Registering TS with master...
19:59:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.418116 8610 external_mini_cluster.cc:1428] Started /tmp/dist-test-task3Meco8/build/asan/bin/kudu as pid 8903
19:59:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.418291 8610 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-0/wal/instance
19:59:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.418828 9063 heartbeater.cc:507] Master 127.8.104.189:34439 requested a full tablet report, sending...
19:59:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.418766 8652 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.104.129:54865
19:59:22.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.420651 8733 ts_manager.cc:194] Registered new tserver with Master: 60624d7b4dcf468da997ac3454bb427a (127.8.104.129:42643)
19:59:22.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.421947 9062 heartbeater.cc:344] Connected to a master server at 127.8.104.188:43579
19:59:22.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.422394 9062 heartbeater.cc:461] Registering TS with master...
19:59:22.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.422817 9062 heartbeater.cc:507] Master 127.8.104.188:43579 requested a full tablet report, sending...
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.423014 8610 external_mini_cluster.cc:1366] Running /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/wal
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/data
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/logs
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/data/info.pb
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.130:0
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.104.130
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.130
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:22.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
19:59:22.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
19:59:22.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
19:59:22.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.424278 8814 ts_manager.cc:194] Registered new tserver with Master: 60624d7b4dcf468da997ac3454bb427a (127.8.104.129:42643)
19:59:22.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.716567 9066 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:59:22.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.716931 9066 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:22.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.717063 9066 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:59:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.724370 9066 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:59:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.724529 9066 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
19:59:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.724591 9066 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
19:59:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.724663 9066 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
19:59:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.724784 9066 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.104.130
19:59:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.725780 8896 catalog_manager.cc:1261] Loaded cluster ID: a1178ae452f34170ababc7c815199e6f
19:59:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.725857 8896 catalog_manager.cc:1554] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159: loading cluster ID for follower catalog manager: success
19:59:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.728317 8896 catalog_manager.cc:1576] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159: acquiring CA information for follower catalog manager: success
19:59:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.730134 8896 catalog_manager.cc:1604] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.733830 9066 tablet_server_runner.cc:78] Tablet server non-default flags:
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/data
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/wal
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.130:0
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/data/info.pb
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.130
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.104.130
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/logs
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 19:43:25 UTC on 24a791456cd2
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7836
19:59:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:22.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.735536 9066 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.737187 9066 file_cache.cc:492] Constructed file cache file cache with capacity 419430
19:59:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.745715 9071 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.745766 9072 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:22.746335 9074 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.747118 9066 server_base.cc:1047] running on GCE node
19:59:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.747659 9066 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:22.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.748541 9066 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:22.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.749735 9066 hybrid_clock.cc:648] HybridClock initialized: now 1756670362749637 us; error 79 us; skew 500 ppm
19:59:22.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.749987 9066 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.751946 9066 webserver.cc:480] Webserver started at http://127.8.104.130:40461/ using document root <none> and password file <none>
19:59:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.752681 9066 fs_manager.cc:362] Metadata directory not provided
19:59:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.752816 9066 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.753204 9066 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
19:59:22.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.755693 9066 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/data/instance:
19:59:22.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88f4731850ec489b8696c9a74004c187"
19:59:22.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:22 on dist-test-slave-7hw0"
19:59:22.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.756325 9066 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/wal/instance:
19:59:22.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88f4731850ec489b8696c9a74004c187"
19:59:22.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:22 on dist-test-slave-7hw0"
19:59:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.761442 9066 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.002s sys 0.004s
19:59:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.764832 9080 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.766198 9066 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.766355 9066 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/data,/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/wal
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88f4731850ec489b8696c9a74004c187"
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:22 on dist-test-slave-7hw0"
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.766536 9066 fs_report.cc:389] FS layout report
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/wal
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/wal
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/data/data
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:22.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.798396 9066 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
19:59:22.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.799252 9066 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:22.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.799554 9066 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:22.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.802585 9066 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:59:22.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.802698 9066 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:59:22.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.802814 9066 ts_tablet_manager.cc:610] Registered 0 tablets
19:59:22.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.802915 9066 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:59:22.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.847883 9066 rpc_server.cc:307] RPC server started. Bound to: 127.8.104.130:34681
19:59:22.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.847929 9201 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.104.130:34681 every 8 connection(s)
19:59:22.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.849277 9066 server_base.cc:1179] Dumped server information to /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/data/info.pb
19:59:22.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.852620 9202 heartbeater.cc:344] Connected to a master server at 127.8.104.190:42679
19:59:22.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.853070 9202 heartbeater.cc:461] Registering TS with master...
19:59:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.857297 9202 heartbeater.cc:507] Master 127.8.104.190:42679 requested a full tablet report, sending...
19:59:22.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.859445 8652 ts_manager.cc:194] Registered new tserver with Master: 88f4731850ec489b8696c9a74004c187 (127.8.104.130:34681)
19:59:22.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.859586 8610 external_mini_cluster.cc:1428] Started /tmp/dist-test-task3Meco8/build/asan/bin/kudu as pid 9066
19:59:22.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.859755 8610 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-1/wal/instance
19:59:22.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.860507 8652 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.104.130:44893
19:59:22.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.862912 9203 heartbeater.cc:344] Connected to a master server at 127.8.104.188:43579
19:59:22.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.863063 9203 heartbeater.cc:461] Registering TS with master...
19:59:22.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.863404 9203 heartbeater.cc:507] Master 127.8.104.188:43579 requested a full tablet report, sending...
19:59:22.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.863835 8610 external_mini_cluster.cc:1366] Running /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:22.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/wal
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/data
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/logs
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/data/info.pb
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.131:0
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.104.131
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.131
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
19:59:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.864327 8814 ts_manager.cc:194] Registered new tserver with Master: 88f4731850ec489b8696c9a74004c187 (127.8.104.130:34681)
19:59:22.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.867837 9204 heartbeater.cc:344] Connected to a master server at 127.8.104.189:34439
19:59:22.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.868001 9204 heartbeater.cc:461] Registering TS with master...
19:59:22.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.868361 9204 heartbeater.cc:507] Master 127.8.104.189:34439 requested a full tablet report, sending...
19:59:22.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:22.869400 8733 ts_manager.cc:194] Registered new tserver with Master: 88f4731850ec489b8696c9a74004c187 (127.8.104.130:34681)
19:59:23.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.022573 8922 catalog_manager.cc:1261] Loaded cluster ID: a1178ae452f34170ababc7c815199e6f
19:59:23.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.022727 8922 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d: loading cluster ID for follower catalog manager: success
19:59:23.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.026244 8922 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d: acquiring CA information for follower catalog manager: success
19:59:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.028640 8922 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:59:23.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.169199 9207 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:59:23.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.169521 9207 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:23.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.169663 9207 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:59:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.177112 9207 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:59:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.177263 9207 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
19:59:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.177332 9207 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
19:59:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.177381 9207 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
19:59:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.177503 9207 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.104.131
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.186376 9207 tablet_server_runner.cc:78] Tablet server non-default flags:
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/data
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/wal
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.131:0
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/data/info.pb
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.131
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:59:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.104.131
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/logs
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 19:43:25 UTC on 24a791456cd2
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7836
19:59:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.188148 9207 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.189900 9207 file_cache.cc:492] Constructed file cache file cache with capacity 419430
19:59:23.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.198508 9214 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:23.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.198628 9213 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.200104 9216 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.200675 9207 server_base.cc:1047] running on GCE node
19:59:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.201308 9207 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:23.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.202466 9207 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:23.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.203709 9207 hybrid_clock.cc:648] HybridClock initialized: now 1756670363203646 us; error 42 us; skew 500 ppm
19:59:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.204073 9207 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.206596 9207 webserver.cc:480] Webserver started at http://127.8.104.131:45373/ using document root <none> and password file <none>
19:59:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.207484 9207 fs_manager.cc:362] Metadata directory not provided
19:59:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.207614 9207 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:23.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.208030 9207 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
19:59:23.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.210983 9207 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/data/instance:
19:59:23.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "92e16bf1ab8c41e09fb4057ecffe50a1"
19:59:23.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:23 on dist-test-slave-7hw0"
19:59:23.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.211928 9207 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/wal/instance:
19:59:23.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "92e16bf1ab8c41e09fb4057ecffe50a1"
19:59:23.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:23 on dist-test-slave-7hw0"
19:59:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.218417 9207 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
19:59:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.222388 9222 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.223976 9207 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.224143 9207 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/data,/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/wal
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "92e16bf1ab8c41e09fb4057ecffe50a1"
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:23 on dist-test-slave-7hw0"
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.224380 9207 fs_report.cc:389] FS layout report
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/wal
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/wal
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/data/data
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:23.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.249600 9207 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
19:59:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.250638 9207 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.250977 9207 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.254734 9207 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:59:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.254853 9207 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:59:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.254961 9207 ts_tablet_manager.cc:610] Registered 0 tablets
19:59:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.255030 9207 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:59:23.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.338505 9207 rpc_server.cc:307] RPC server started. Bound to: 127.8.104.131:42283
19:59:23.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.338599 9343 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.104.131:42283 every 8 connection(s)
19:59:23.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.339834 9207 server_base.cc:1179] Dumped server information to /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/data/info.pb
19:59:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.342375 9344 heartbeater.cc:344] Connected to a master server at 127.8.104.190:42679
19:59:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.342778 9344 heartbeater.cc:461] Registering TS with master...
19:59:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.344048 9344 heartbeater.cc:507] Master 127.8.104.190:42679 requested a full tablet report, sending...
19:59:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.345587 8610 external_mini_cluster.cc:1428] Started /tmp/dist-test-task3Meco8/build/asan/bin/kudu as pid 9207
19:59:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.345804 8610 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/ts-2/wal/instance
19:59:23.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.351052 9345 heartbeater.cc:344] Connected to a master server at 127.8.104.188:43579
19:59:23.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.351222 9345 heartbeater.cc:461] Registering TS with master...
19:59:23.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.351764 9345 heartbeater.cc:507] Master 127.8.104.188:43579 requested a full tablet report, sending...
19:59:23.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.353145 8813 ts_manager.cc:194] Registered new tserver with Master: 92e16bf1ab8c41e09fb4057ecffe50a1 (127.8.104.131:42283)
19:59:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.356761 8651 ts_manager.cc:194] Registered new tserver with Master: 92e16bf1ab8c41e09fb4057ecffe50a1 (127.8.104.131:42283)
19:59:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.357844 8651 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.104.131:44171
19:59:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.360875 9346 heartbeater.cc:344] Connected to a master server at 127.8.104.189:34439
19:59:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.361002 9346 heartbeater.cc:461] Registering TS with master...
19:59:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.361304 9346 heartbeater.cc:507] Master 127.8.104.189:34439 requested a full tablet report, sending...
19:59:23.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.362397 8733 ts_manager.cc:194] Registered new tserver with Master: 92e16bf1ab8c41e09fb4057ecffe50a1 (127.8.104.131:42283)
19:59:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.374558 8610 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
19:59:23.376 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:59:23.376 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
19:59:23.377 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.104.190"
port: 42679
}
bound_http_address {
host: "127.8.104.190"
port: 46569
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.104.189"
port: 34439
}
bound_http_address {
host: "127.8.104.189"
port: 33457
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.104.188"
port: 43579
}
bound_http_address {
host: "127.8.104.188"
port: 38037
}
}
}
19:59:23.377 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:59:23.378 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.8.104.129"
port: 42643
}
bound_http_address {
host: "127.8.104.129"
port: 39951
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.8.104.130"
port: 34681
}
bound_http_address {
host: "127.8.104.130"
port: 40461
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.8.104.131"
port: 42283
}
bound_http_address {
host: "127.8.104.131"
port: 45373
}
}
}
19:59:23.378 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
19:59:23.380 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
19:59:23.381 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.104.190:42679(127.8.104.190:42679)] connecting to peer
19:59:23.383 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:23.384 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:23.394 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.104.190:42679(127.8.104.190:42679)] Successfully connected to peer
19:59:23.395 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.104.189:34439(127.8.104.189:34439)] Successfully connected to peer
19:59:23.395 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.8.104.188:43579(127.8.104.188:43579)] Successfully connected to peer
19:59:23.397 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.8.104.188
19:59:23.399 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.8.104.189
19:59:23.404 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.8.104.190
19:59:23.410 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x22b99fae, L:/127.0.0.1:54940 - R:/127.8.104.188:43579] using SASL/PLAIN
19:59:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.423478 9061 heartbeater.cc:499] Master 127.8.104.190:42679 was elected leader, sending a full tablet report...
19:59:23.426 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa51a4dec, L:/127.0.0.1:60068 - R:/127.8.104.190:42679] using SASL/PLAIN
19:59:23.427 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x61d09adf, L:/127.0.0.1:41032 - R:/127.8.104.189:34439] using SASL/PLAIN
19:59:23.428 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
19:59:23.429 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=48), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [31ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, deferred=null) after lookup
19:59:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.430464 8651 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60068:
19:59:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
19:59:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
19:59:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
19:59:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
19:59:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
19:59:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.432650 8651 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
19:59:23.445 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 3c83f2cf970e4d34aa8a7838bb1701ab
19:59:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.494202 8995 tablet_service.cc:1468] Processing CreateTablet for tablet b95b19d800b8486ab36c1352b8e6ba74 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
19:59:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.494203 9136 tablet_service.cc:1468] Processing CreateTablet for tablet b95b19d800b8486ab36c1352b8e6ba74 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
19:59:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.494848 9137 tablet_service.cc:1468] Processing CreateTablet for tablet e00e833ec2d443a68460764399105faf (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
19:59:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.494390 8996 tablet_service.cc:1468] Processing CreateTablet for tablet e00e833ec2d443a68460764399105faf (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
19:59:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.495620 8995 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b95b19d800b8486ab36c1352b8e6ba74. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.495651 9136 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b95b19d800b8486ab36c1352b8e6ba74. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:23.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.496131 9137 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e00e833ec2d443a68460764399105faf. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:23.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.497263 8996 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e00e833ec2d443a68460764399105faf. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:23.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.501261 9278 tablet_service.cc:1468] Processing CreateTablet for tablet b95b19d800b8486ab36c1352b8e6ba74 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
19:59:23.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.502732 9278 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b95b19d800b8486ab36c1352b8e6ba74. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.509976 9279 tablet_service.cc:1468] Processing CreateTablet for tablet e00e833ec2d443a68460764399105faf (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
19:59:23.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.510469 9279 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e00e833ec2d443a68460764399105faf. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:23.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.514676 9369 tablet_bootstrap.cc:492] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187: Bootstrap starting.
19:59:23.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.516571 9370 tablet_bootstrap.cc:492] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a: Bootstrap starting.
19:59:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.519013 9369 tablet_bootstrap.cc:654] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187: Neither blocks nor log segments found. Creating new log.
19:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.520161 9370 tablet_bootstrap.cc:654] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a: Neither blocks nor log segments found. Creating new log.
19:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.520363 9369 log.cc:826] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187: Log is configured to *not* fsync() on all Append() calls
19:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.521205 9371 tablet_bootstrap.cc:492] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1: Bootstrap starting.
19:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.521653 9370 log.cc:826] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a: Log is configured to *not* fsync() on all Append() calls
19:59:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.524276 9369 tablet_bootstrap.cc:492] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187: No bootstrap required, opened a new log
19:59:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.524657 9371 tablet_bootstrap.cc:654] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1: Neither blocks nor log segments found. Creating new log.
19:59:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.525069 9369 ts_tablet_manager.cc:1397] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187: Time spent bootstrapping tablet: real 0.011s user 0.008s sys 0.000s
19:59:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.525363 9370 tablet_bootstrap.cc:492] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a: No bootstrap required, opened a new log
19:59:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.525611 9370 ts_tablet_manager.cc:1397] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a: Time spent bootstrapping tablet: real 0.009s user 0.009s sys 0.000s
19:59:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.525808 9371 log.cc:826] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1: Log is configured to *not* fsync() on all Append() calls
19:59:23.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.528245 9371 tablet_bootstrap.cc:492] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1: No bootstrap required, opened a new log
19:59:23.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.528465 9371 ts_tablet_manager.cc:1397] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.003s
19:59:23.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.530676 9370 raft_consensus.cc:357] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.531077 9370 raft_consensus.cc:383] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:23.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.531193 9370 raft_consensus.cc:738] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 60624d7b4dcf468da997ac3454bb427a, State: Initialized, Role: FOLLOWER
19:59:23.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.531065 9369 raft_consensus.cc:357] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.531373 9369 raft_consensus.cc:383] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:23.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.531682 9369 raft_consensus.cc:738] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 88f4731850ec489b8696c9a74004c187, State: Initialized, Role: FOLLOWER
19:59:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.531852 9370 consensus_queue.cc:260] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.532366 9369 consensus_queue.cc:260] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.533510 9370 ts_tablet_manager.cc:1428] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a: Time spent starting tablet: real 0.008s user 0.007s sys 0.000s
19:59:23.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.535888 9371 raft_consensus.cc:357] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.536319 9371 raft_consensus.cc:383] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:23.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.536487 9370 tablet_bootstrap.cc:492] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a: Bootstrap starting.
19:59:23.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.536469 9371 raft_consensus.cc:738] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 92e16bf1ab8c41e09fb4057ecffe50a1, State: Initialized, Role: FOLLOWER
19:59:23.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.537169 9371 consensus_queue.cc:260] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.538738 9370 tablet_bootstrap.cc:654] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a: Neither blocks nor log segments found. Creating new log.
19:59:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.546861 9370 tablet_bootstrap.cc:492] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a: No bootstrap required, opened a new log
19:59:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.547219 9370 ts_tablet_manager.cc:1397] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.010s
19:59:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.548126 9370 raft_consensus.cc:357] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.548413 9370 raft_consensus.cc:383] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.548475 9344 heartbeater.cc:499] Master 127.8.104.190:42679 was elected leader, sending a full tablet report...
19:59:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.548604 9370 raft_consensus.cc:738] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 60624d7b4dcf468da997ac3454bb427a, State: Initialized, Role: FOLLOWER
19:59:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.548764 9371 ts_tablet_manager.cc:1428] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1: Time spent starting tablet: real 0.020s user 0.016s sys 0.005s
19:59:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.548759 9369 ts_tablet_manager.cc:1428] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187: Time spent starting tablet: real 0.024s user 0.019s sys 0.004s
19:59:23.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.549083 9370 consensus_queue.cc:260] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.549219 9371 tablet_bootstrap.cc:492] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1: Bootstrap starting.
19:59:23.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.549666 9370 ts_tablet_manager.cc:1428] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
19:59:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.551017 9371 tablet_bootstrap.cc:654] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1: Neither blocks nor log segments found. Creating new log.
19:59:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.554211 9202 heartbeater.cc:499] Master 127.8.104.190:42679 was elected leader, sending a full tablet report...
19:59:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.554989 9371 tablet_bootstrap.cc:492] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1: No bootstrap required, opened a new log
19:59:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.555141 9371 ts_tablet_manager.cc:1397] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.003s
19:59:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.556066 9371 raft_consensus.cc:357] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.556249 9371 raft_consensus.cc:383] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.556314 9371 raft_consensus.cc:738] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 92e16bf1ab8c41e09fb4057ecffe50a1, State: Initialized, Role: FOLLOWER
19:59:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.556542 9371 consensus_queue.cc:260] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.556981 9371 ts_tablet_manager.cc:1428] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
19:59:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.557312 9369 tablet_bootstrap.cc:492] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187: Bootstrap starting.
19:59:23.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.559167 9369 tablet_bootstrap.cc:654] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187: Neither blocks nor log segments found. Creating new log.
19:59:23.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.562232 9369 tablet_bootstrap.cc:492] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187: No bootstrap required, opened a new log
19:59:23.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.562506 9369 ts_tablet_manager.cc:1397] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
19:59:23.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.563411 9369 raft_consensus.cc:357] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.563622 9369 raft_consensus.cc:383] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:23.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.563702 9369 raft_consensus.cc:738] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 88f4731850ec489b8696c9a74004c187, State: Initialized, Role: FOLLOWER
19:59:23.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.563922 9369 consensus_queue.cc:260] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.564358 9369 ts_tablet_manager.cc:1428] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
19:59:23.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.605787 9347 tablet.cc:2378] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:59:23.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.616977 9205 tablet.cc:2378] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:59:23.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.637334 9375 raft_consensus.cc:491] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:59:23.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.637609 9375 raft_consensus.cc:513] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.639317 9375 leader_election.cc:290] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88f4731850ec489b8696c9a74004c187 (127.8.104.130:34681), 92e16bf1ab8c41e09fb4057ecffe50a1 (127.8.104.131:42283)
19:59:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.646389 9299 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e00e833ec2d443a68460764399105faf" candidate_uuid: "60624d7b4dcf468da997ac3454bb427a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" is_pre_election: true
19:59:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.646441 9157 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e00e833ec2d443a68460764399105faf" candidate_uuid: "60624d7b4dcf468da997ac3454bb427a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "88f4731850ec489b8696c9a74004c187" is_pre_election: true
19:59:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.646814 9299 raft_consensus.cc:2466] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 60624d7b4dcf468da997ac3454bb427a in term 0.
19:59:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.646840 9157 raft_consensus.cc:2466] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 60624d7b4dcf468da997ac3454bb427a in term 0.
19:59:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.647389 8941 leader_election.cc:304] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 60624d7b4dcf468da997ac3454bb427a, 88f4731850ec489b8696c9a74004c187, 92e16bf1ab8c41e09fb4057ecffe50a1; no voters:
19:59:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.647720 9375 raft_consensus.cc:2802] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Leader pre-election won for term 1
19:59:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.647861 9375 raft_consensus.cc:491] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:59:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.647948 9375 raft_consensus.cc:3058] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Advancing to term 1
19:59:23.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.649849 9375 raft_consensus.cc:513] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.650358 9375 leader_election.cc:290] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [CANDIDATE]: Term 1 election: Requested vote from peers 88f4731850ec489b8696c9a74004c187 (127.8.104.130:34681), 92e16bf1ab8c41e09fb4057ecffe50a1 (127.8.104.131:42283)
19:59:23.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.650746 9157 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e00e833ec2d443a68460764399105faf" candidate_uuid: "60624d7b4dcf468da997ac3454bb427a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "88f4731850ec489b8696c9a74004c187"
19:59:23.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.650907 9157 raft_consensus.cc:3058] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Advancing to term 1
19:59:23.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.650869 9299 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "e00e833ec2d443a68460764399105faf" candidate_uuid: "60624d7b4dcf468da997ac3454bb427a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1"
19:59:23.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.651069 9299 raft_consensus.cc:3058] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Advancing to term 1
19:59:23.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.652781 9157 raft_consensus.cc:2466] T e00e833ec2d443a68460764399105faf P 88f4731850ec489b8696c9a74004c187 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 60624d7b4dcf468da997ac3454bb427a in term 1.
19:59:23.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.653082 9299 raft_consensus.cc:2466] T e00e833ec2d443a68460764399105faf P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 60624d7b4dcf468da997ac3454bb427a in term 1.
19:59:23.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.653193 8939 leader_election.cc:304] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 60624d7b4dcf468da997ac3454bb427a, 88f4731850ec489b8696c9a74004c187; no voters:
19:59:23.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.653493 9375 raft_consensus.cc:2802] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [term 1 FOLLOWER]: Leader election won for term 1
19:59:23.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.653952 9375 raft_consensus.cc:695] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [term 1 LEADER]: Becoming Leader. State: Replica: 60624d7b4dcf468da997ac3454bb427a, State: Running, Role: LEADER
19:59:23.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.654636 9375 consensus_queue.cc:237] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.659538 8652 catalog_manager.cc:5582] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a reported cstate change: term changed from 0 to 1, leader changed from <none> to 60624d7b4dcf468da997ac3454bb427a (127.8.104.129). New cstate: current_term: 1 leader_uuid: "60624d7b4dcf468da997ac3454bb427a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } health_report { overall_health: UNKNOWN } } }
19:59:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.663389 9064 tablet.cc:2378] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:59:23.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.727672 9376 raft_consensus.cc:491] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:59:23.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.727878 9376 raft_consensus.cc:513] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.729573 9376 leader_election.cc:290] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 60624d7b4dcf468da997ac3454bb427a (127.8.104.129:42643), 92e16bf1ab8c41e09fb4057ecffe50a1 (127.8.104.131:42283)
19:59:23.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.736299 9299 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b95b19d800b8486ab36c1352b8e6ba74" candidate_uuid: "88f4731850ec489b8696c9a74004c187" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" is_pre_election: true
19:59:23.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.736542 9299 raft_consensus.cc:2466] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 88f4731850ec489b8696c9a74004c187 in term 0.
19:59:23.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.736991 9084 leader_election.cc:304] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 88f4731850ec489b8696c9a74004c187, 92e16bf1ab8c41e09fb4057ecffe50a1; no voters:
19:59:23.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.737099 9016 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b95b19d800b8486ab36c1352b8e6ba74" candidate_uuid: "88f4731850ec489b8696c9a74004c187" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "60624d7b4dcf468da997ac3454bb427a" is_pre_election: true
19:59:23.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.737412 9376 raft_consensus.cc:2802] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Leader pre-election won for term 1
19:59:23.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.737457 9016 raft_consensus.cc:2466] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 88f4731850ec489b8696c9a74004c187 in term 0.
19:59:23.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.737560 9376 raft_consensus.cc:491] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:59:23.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.737689 9376 raft_consensus.cc:3058] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Advancing to term 1
19:59:23.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.739583 9376 raft_consensus.cc:513] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.740149 9376 leader_election.cc:290] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [CANDIDATE]: Term 1 election: Requested vote from peers 60624d7b4dcf468da997ac3454bb427a (127.8.104.129:42643), 92e16bf1ab8c41e09fb4057ecffe50a1 (127.8.104.131:42283)
19:59:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.740586 9016 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b95b19d800b8486ab36c1352b8e6ba74" candidate_uuid: "88f4731850ec489b8696c9a74004c187" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "60624d7b4dcf468da997ac3454bb427a"
19:59:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.740571 9299 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "b95b19d800b8486ab36c1352b8e6ba74" candidate_uuid: "88f4731850ec489b8696c9a74004c187" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1"
19:59:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.740746 9016 raft_consensus.cc:3058] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Advancing to term 1
19:59:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.740768 9299 raft_consensus.cc:3058] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Advancing to term 1
19:59:23.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.743384 9299 raft_consensus.cc:2466] T b95b19d800b8486ab36c1352b8e6ba74 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 88f4731850ec489b8696c9a74004c187 in term 1.
19:59:23.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.743506 9016 raft_consensus.cc:2466] T b95b19d800b8486ab36c1352b8e6ba74 P 60624d7b4dcf468da997ac3454bb427a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 88f4731850ec489b8696c9a74004c187 in term 1.
19:59:23.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.743788 9084 leader_election.cc:304] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 88f4731850ec489b8696c9a74004c187, 92e16bf1ab8c41e09fb4057ecffe50a1; no voters:
19:59:23.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.744058 9376 raft_consensus.cc:2802] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [term 1 FOLLOWER]: Leader election won for term 1
19:59:23.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.744462 9376 raft_consensus.cc:695] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [term 1 LEADER]: Becoming Leader. State: Replica: 88f4731850ec489b8696c9a74004c187, State: Running, Role: LEADER
19:59:23.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.744940 9376 consensus_queue.cc:237] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.749238 8652 catalog_manager.cc:5582] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 reported cstate change: term changed from 0 to 1, leader changed from <none> to 88f4731850ec489b8696c9a74004c187 (127.8.104.130). New cstate: current_term: 1 leader_uuid: "88f4731850ec489b8696c9a74004c187" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } health_report { overall_health: UNKNOWN } } }
19:59:23.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.924649 8652 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60082:
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
19:59:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:23.925833 8652 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
19:59:23.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.936295 8995 tablet_service.cc:1468] Processing CreateTablet for tablet 22590beba9194de0adb105319747e148 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d80f4524b53242f180d0127d58b6d72b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
19:59:23.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.936650 9137 tablet_service.cc:1468] Processing CreateTablet for tablet 22590beba9194de0adb105319747e148 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d80f4524b53242f180d0127d58b6d72b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
19:59:23.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.937191 8995 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 22590beba9194de0adb105319747e148. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:23.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.937470 9137 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 22590beba9194de0adb105319747e148. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:23.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.942207 9370 tablet_bootstrap.cc:492] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: Bootstrap starting.
19:59:23.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.942090 9279 tablet_service.cc:1468] Processing CreateTablet for tablet 22590beba9194de0adb105319747e148 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d80f4524b53242f180d0127d58b6d72b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
19:59:23.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.943071 9279 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 22590beba9194de0adb105319747e148. 1 dirs total, 0 dirs full, 0 dirs failed
19:59:23.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.943838 9370 tablet_bootstrap.cc:654] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: Neither blocks nor log segments found. Creating new log.
19:59:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.946192 9369 tablet_bootstrap.cc:492] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187: Bootstrap starting.
19:59:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.946640 9370 tablet_bootstrap.cc:492] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: No bootstrap required, opened a new log
19:59:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.946770 9370 ts_tablet_manager.cc:1397] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
19:59:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.947474 9370 raft_consensus.cc:357] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.947636 9370 raft_consensus.cc:383] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.947713 9370 raft_consensus.cc:738] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 60624d7b4dcf468da997ac3454bb427a, State: Initialized, Role: FOLLOWER
19:59:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.947782 9369 tablet_bootstrap.cc:654] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187: Neither blocks nor log segments found. Creating new log.
19:59:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.947974 9370 consensus_queue.cc:260] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.948356 9375 tablet_replica.cc:440] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } } }
19:59:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.948582 9375 tablet_replica.cc:443] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: This TxnStatusTablet replica's current role is: FOLLOWER
19:59:23.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.949649 9371 tablet_bootstrap.cc:492] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1: Bootstrap starting.
19:59:23.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.951254 9371 tablet_bootstrap.cc:654] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1: Neither blocks nor log segments found. Creating new log.
19:59:23.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.951288 9370 ts_tablet_manager.cc:1428] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
19:59:23.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.955300 9369 tablet_bootstrap.cc:492] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187: No bootstrap required, opened a new log
19:59:23.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.955420 9369 ts_tablet_manager.cc:1397] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.004s
19:59:23.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.955435 9371 tablet_bootstrap.cc:492] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1: No bootstrap required, opened a new log
19:59:23.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.955600 9371 ts_tablet_manager.cc:1397] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.001s
19:59:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.955991 9369 raft_consensus.cc:357] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.956122 9369 raft_consensus.cc:383] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.956177 9369 raft_consensus.cc:738] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 88f4731850ec489b8696c9a74004c187, State: Initialized, Role: FOLLOWER
19:59:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.956202 9371 raft_consensus.cc:357] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.956373 9371 raft_consensus.cc:383] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:59:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.956363 9369 consensus_queue.cc:260] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.956458 9371 raft_consensus.cc:738] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 92e16bf1ab8c41e09fb4057ecffe50a1, State: Initialized, Role: FOLLOWER
19:59:23.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.956727 9376 tablet_replica.cc:440] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } } }
19:59:23.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.956964 9376 tablet_replica.cc:443] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187: This TxnStatusTablet replica's current role is: FOLLOWER
19:59:23.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.957656 9369 ts_tablet_manager.cc:1428] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
19:59:23.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.957238 9371 consensus_queue.cc:260] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.959151 9377 tablet_replica.cc:440] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } } }
19:59:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.959404 9377 tablet_replica.cc:443] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1: This TxnStatusTablet replica's current role is: FOLLOWER
19:59:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:23.962257 9371 ts_tablet_manager.cc:1428] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1: Time spent starting tablet: real 0.007s user 0.001s sys 0.001s
19:59:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.089805 9375 consensus_queue.cc:1035] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [LEADER]: Connected to new peer: Peer: permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:59:24.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.099165 9402 consensus_queue.cc:1035] T e00e833ec2d443a68460764399105faf P 60624d7b4dcf468da997ac3454bb427a [LEADER]: Connected to new peer: Peer: permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:59:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.100018 9375 raft_consensus.cc:491] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:59:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.100138 9375 raft_consensus.cc:513] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.100617 9375 leader_election.cc:290] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88f4731850ec489b8696c9a74004c187 (127.8.104.130:34681), 92e16bf1ab8c41e09fb4057ecffe50a1 (127.8.104.131:42283)
19:59:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.101105 9157 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "22590beba9194de0adb105319747e148" candidate_uuid: "60624d7b4dcf468da997ac3454bb427a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "88f4731850ec489b8696c9a74004c187" is_pre_election: true
19:59:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.101317 9157 raft_consensus.cc:2466] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 60624d7b4dcf468da997ac3454bb427a in term 0.
19:59:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.101697 8939 leader_election.cc:304] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 60624d7b4dcf468da997ac3454bb427a, 88f4731850ec489b8696c9a74004c187; no voters:
19:59:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.101929 9402 raft_consensus.cc:2802] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Leader pre-election won for term 1
19:59:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.102116 9402 raft_consensus.cc:491] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:59:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.102228 9402 raft_consensus.cc:3058] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [term 0 FOLLOWER]: Advancing to term 1
19:59:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.102634 9298 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "22590beba9194de0adb105319747e148" candidate_uuid: "60624d7b4dcf468da997ac3454bb427a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" is_pre_election: true
19:59:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.102887 9298 raft_consensus.cc:2466] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 60624d7b4dcf468da997ac3454bb427a in term 0.
19:59:24.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.104974 9402 raft_consensus.cc:513] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.105952 9157 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "22590beba9194de0adb105319747e148" candidate_uuid: "60624d7b4dcf468da997ac3454bb427a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "88f4731850ec489b8696c9a74004c187"
19:59:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.106155 9157 raft_consensus.cc:3058] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187 [term 0 FOLLOWER]: Advancing to term 1
19:59:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.108632 9157 raft_consensus.cc:2466] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 60624d7b4dcf468da997ac3454bb427a in term 1.
19:59:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.109237 8939 leader_election.cc:304] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 60624d7b4dcf468da997ac3454bb427a, 88f4731850ec489b8696c9a74004c187; no voters:
19:59:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.110163 9402 leader_election.cc:290] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [CANDIDATE]: Term 1 election: Requested vote from peers 88f4731850ec489b8696c9a74004c187 (127.8.104.130:34681), 92e16bf1ab8c41e09fb4057ecffe50a1 (127.8.104.131:42283)
19:59:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.110241 9299 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "22590beba9194de0adb105319747e148" candidate_uuid: "60624d7b4dcf468da997ac3454bb427a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1"
19:59:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.110385 9375 raft_consensus.cc:2802] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [term 1 FOLLOWER]: Leader election won for term 1
19:59:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.110525 9375 raft_consensus.cc:695] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [term 1 LEADER]: Becoming Leader. State: Replica: 60624d7b4dcf468da997ac3454bb427a, State: Running, Role: LEADER
19:59:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.110679 9299 raft_consensus.cc:3058] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 0 FOLLOWER]: Advancing to term 1
19:59:24.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.110792 9375 consensus_queue.cc:237] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } }
19:59:24.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.112982 9299 raft_consensus.cc:2466] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 60624d7b4dcf468da997ac3454bb427a in term 1.
19:59:24.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.113222 9402 tablet_replica.cc:440] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: TxnStatusTablet state changed. Reason: New leader 60624d7b4dcf468da997ac3454bb427a. Latest consensus state: current_term: 1 leader_uuid: "60624d7b4dcf468da997ac3454bb427a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } } }
19:59:24.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.113646 9402 tablet_replica.cc:443] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: This TxnStatusTablet replica's current role is: LEADER
19:59:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.113976 8652 catalog_manager.cc:5582] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a reported cstate change: term changed from 0 to 1, leader changed from <none> to 60624d7b4dcf468da997ac3454bb427a (127.8.104.129). New cstate: current_term: 1 leader_uuid: "60624d7b4dcf468da997ac3454bb427a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } health_report { overall_health: UNKNOWN } } }
19:59:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.114360 9407 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
19:59:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.114586 9407 txn_status_manager.cc:930] Loading transaction status metadata into memory...
19:59:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.143486 9157 raft_consensus.cc:1273] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187 [term 1 FOLLOWER]: Refusing update from remote peer 60624d7b4dcf468da997ac3454bb427a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:59:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.143476 9299 raft_consensus.cc:1273] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1 [term 1 FOLLOWER]: Refusing update from remote peer 60624d7b4dcf468da997ac3454bb427a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:59:24.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.144119 9375 consensus_queue.cc:1035] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [LEADER]: Connected to new peer: Peer: permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:59:24.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.144389 9402 consensus_queue.cc:1035] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a [LEADER]: Connected to new peer: Peer: permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:59:24.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.154217 9376 tablet_replica.cc:440] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187: TxnStatusTablet state changed. Reason: New leader 60624d7b4dcf468da997ac3454bb427a. Latest consensus state: current_term: 1 leader_uuid: "60624d7b4dcf468da997ac3454bb427a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } } }
19:59:24.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.154428 9376 tablet_replica.cc:443] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187: This TxnStatusTablet replica's current role is: FOLLOWER
19:59:24.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.155205 9377 tablet_replica.cc:440] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1: TxnStatusTablet state changed. Reason: New leader 60624d7b4dcf468da997ac3454bb427a. Latest consensus state: current_term: 1 leader_uuid: "60624d7b4dcf468da997ac3454bb427a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } } }
19:59:24.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.155421 9377 tablet_replica.cc:443] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1: This TxnStatusTablet replica's current role is: FOLLOWER
19:59:24.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.159392 9401 mvcc.cc:204] Tried to move back new op lower bound from 7195321811525234688 to 7195321811399778304. Current Snapshot: MvccSnapshot[applied={T|T < 7195321811525234688}]
19:59:24.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.167235 9402 tablet_replica.cc:440] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "60624d7b4dcf468da997ac3454bb427a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } } }
19:59:24.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.167456 9402 tablet_replica.cc:443] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: This TxnStatusTablet replica's current role is: LEADER
19:59:24.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.167690 9375 tablet_replica.cc:440] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "60624d7b4dcf468da997ac3454bb427a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } } }
19:59:24.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.167855 9375 tablet_replica.cc:443] T 22590beba9194de0adb105319747e148 P 60624d7b4dcf468da997ac3454bb427a: This TxnStatusTablet replica's current role is: LEADER
19:59:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.173684 9376 tablet_replica.cc:440] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "60624d7b4dcf468da997ac3454bb427a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } } }
19:59:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.173900 9376 tablet_replica.cc:443] T 22590beba9194de0adb105319747e148 P 88f4731850ec489b8696c9a74004c187: This TxnStatusTablet replica's current role is: FOLLOWER
19:59:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.174374 9405 mvcc.cc:204] Tried to move back new op lower bound from 7195321811525234688 to 7195321811399778304. Current Snapshot: MvccSnapshot[applied={T|T < 7195321811525234688}]
19:59:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.176112 9376 consensus_queue.cc:1035] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [LEADER]: Connected to new peer: Peer: permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:59:24.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.177652 9377 tablet_replica.cc:440] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "60624d7b4dcf468da997ac3454bb427a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" member_type: VOTER last_known_addr { host: "127.8.104.129" port: 42643 } } peers { permanent_uuid: "88f4731850ec489b8696c9a74004c187" member_type: VOTER last_known_addr { host: "127.8.104.130" port: 34681 } } peers { permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 } } }
19:59:24.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.177861 9377 tablet_replica.cc:443] T 22590beba9194de0adb105319747e148 P 92e16bf1ab8c41e09fb4057ecffe50a1: This TxnStatusTablet replica's current role is: FOLLOWER
19:59:24.180 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
19:59:24.182 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 3c83f2cf970e4d34aa8a7838bb1701ab
19:59:24.183 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.104.190:42679
19:59:24.183 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
19:59:24.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.186084 8610 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task3Meco8/build/asan/bin/kudu pid 8621
19:59:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.187372 8692 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.188335 9387 consensus_queue.cc:1035] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 [LEADER]: Connected to new peer: Peer: permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" member_type: VOTER last_known_addr { host: "127.8.104.131" port: 42283 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:59:24.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.313671 8610 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task3Meco8/build/asan/bin/kudu with pid 8621
19:59:24.335 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.104.190:42679(127.8.104.190:42679)] handling channelInactive
19:59:24.335 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.190:42679(127.8.104.190:42679)] cleaning up while in state READY due to: connection closed
19:59:24.335 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:59:24.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:24.336225 9224 connection.cc:537] client connection to 127.8.104.190:42679 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
19:59:24.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:24.336546 9061 heartbeater.cc:646] Failed to heartbeat to 127.8.104.190:42679 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.8.104.190:42679 (error 108)
19:59:24.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:24.336702 9344 heartbeater.cc:646] Failed to heartbeat to 127.8.104.190:42679 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
19:59:24.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.612511 9418 raft_consensus.cc:491] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader e11e01529b2f4bf88c980fdc7cc21f92)
19:59:24.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.612634 9419 raft_consensus.cc:491] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader e11e01529b2f4bf88c980fdc7cc21f92)
19:59:24.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.612713 9418 raft_consensus.cc:513] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:24.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.612874 9419 raft_consensus.cc:513] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:24.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.613960 9418 leader_election.cc:290] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:24.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.614104 9419 leader_election.cc:290] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679), 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439)
19:59:24.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.614537 8824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "89ecc82c1296466abc4576789e40284d" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" is_pre_election: true
19:59:24.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.615021 8824 raft_consensus.cc:2466] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 89ecc82c1296466abc4576789e40284d in term 1.
19:59:24.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:24.615355 8799 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111)
19:59:24.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.615577 8720 leader_election.cc:304] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159; no voters:
19:59:24.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.616168 9418 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 1 FOLLOWER]: Leader pre-election won for term 2
19:59:24.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.616389 9418 raft_consensus.cc:491] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 1 FOLLOWER]: Starting leader election (detected failure of leader e11e01529b2f4bf88c980fdc7cc21f92)
19:59:24.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.616568 8742 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "89ecc82c1296466abc4576789e40284d" is_pre_election: true
19:59:24.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.616765 9418 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 1 FOLLOWER]: Advancing to term 2
19:59:24.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:24.617022 8718 leader_election.cc:336] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679): Network error: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111)
19:59:24.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:24.617985 8799 leader_election.cc:336] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679): Network error: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111)
19:59:24.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.619051 9418 raft_consensus.cc:513] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:24.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.619418 8742 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate fa8c3788cc4a40c5bb7b2497e6bab159 in current term 2: Already voted for candidate 89ecc82c1296466abc4576789e40284d in this term.
19:59:24.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.619731 9418 leader_election.cc:290] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [CANDIDATE]: Term 2 election: Requested vote from peers e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:24.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.620095 8799 leader_election.cc:304] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fa8c3788cc4a40c5bb7b2497e6bab159; no voters: 89ecc82c1296466abc4576789e40284d, e11e01529b2f4bf88c980fdc7cc21f92
19:59:24.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.620209 8824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "89ecc82c1296466abc4576789e40284d" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159"
19:59:24.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.620383 8824 raft_consensus.cc:3058] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 1 FOLLOWER]: Advancing to term 2
19:59:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:24.620898 8718 leader_election.cc:336] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679): Network error: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111)
19:59:24.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.622340 8824 raft_consensus.cc:2466] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 89ecc82c1296466abc4576789e40284d in term 2.
19:59:24.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.622493 9419 raft_consensus.cc:2747] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
19:59:24.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.622752 8720 leader_election.cc:304] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159; no voters: e11e01529b2f4bf88c980fdc7cc21f92
19:59:24.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.623023 9418 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 2 FOLLOWER]: Leader election won for term 2
19:59:24.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.623440 9418 raft_consensus.cc:695] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [term 2 LEADER]: Becoming Leader. State: Replica: 89ecc82c1296466abc4576789e40284d, State: Running, Role: LEADER
19:59:24.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.623819 9418 consensus_queue.cc:237] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:24.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.625007 9424 sys_catalog.cc:455] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 89ecc82c1296466abc4576789e40284d. Latest consensus state: current_term: 2 leader_uuid: "89ecc82c1296466abc4576789e40284d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:24.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.625219 9424 sys_catalog.cc:458] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [sys.catalog]: This master's current role is: LEADER
19:59:24.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.625710 9425 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
19:59:24.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.628068 9425 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab]
19:59:24.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.628522 9425 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=d80f4524b53242f180d0127d58b6d72b]
19:59:24.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.630060 9425 tablet_loader.cc:96] loaded metadata for tablet 22590beba9194de0adb105319747e148 (table kudu_system.kudu_transactions [id=d80f4524b53242f180d0127d58b6d72b])
19:59:24.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.630517 9425 tablet_loader.cc:96] loaded metadata for tablet b95b19d800b8486ab36c1352b8e6ba74 (table txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab])
19:59:24.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.630900 9425 tablet_loader.cc:96] loaded metadata for tablet e00e833ec2d443a68460764399105faf (table txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab])
19:59:24.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.631143 9425 catalog_manager.cc:1486] Initializing Kudu cluster ID...
19:59:24.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.631999 9425 catalog_manager.cc:1261] Loaded cluster ID: a1178ae452f34170ababc7c815199e6f
19:59:24.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.632091 9425 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
19:59:24.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.633189 9425 catalog_manager.cc:1506] Loading token signing keys...
19:59:24.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.634055 9425 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d: Loaded TSK: 0
19:59:24.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.634864 9425 catalog_manager.cc:1516] Initializing in-progress tserver states...
19:59:24.682 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.190:42679(127.8.104.190:42679)] connecting to peer
19:59:24.683 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.190:42679(127.8.104.190:42679): Connection refused: /127.8.104.190:42679
19:59:24.683 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.190:42679(127.8.104.190:42679)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.190:42679(127.8.104.190:42679): Connection refused: /127.8.104.190:42679
19:59:24.683 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.8.104.190:42679(127.8.104.190:42679) for tablet Kudu Master: Failed to connect to peer master-127.8.104.190:42679(127.8.104.190:42679): Connection refused: /127.8.104.190:42679
19:59:24.702 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.190:42679(127.8.104.190:42679)] connecting to peer
19:59:24.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:24.702674 8718 consensus_peers.cc:489] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d -> Peer e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679): Couldn't send request to peer e11e01529b2f4bf88c980fdc7cc21f92. Status: Network error: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:59:24.703 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.190:42679(127.8.104.190:42679): Connection refused: /127.8.104.190:42679
19:59:24.703 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.190:42679(127.8.104.190:42679)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.190:42679(127.8.104.190:42679): Connection refused: /127.8.104.190:42679
19:59:24.703 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.190:42679: Failed to connect to peer master-127.8.104.190:42679(127.8.104.190:42679): Connection refused: /127.8.104.190:42679
19:59:24.705 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
19:59:24.706 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=24), Traces: [0ms] sending RPC to server master-127.8.104.190:42679, [2ms] received response from server master-127.8.104.190:42679: Network error: Failed to connect to peer master-127.8.104.190:42679(127.8.104.190:42679): Connection refused: /127.8.104.190:42679, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.8.104.190:42679(127.8.104.190:42679): Connection refused: /127.8.104.190:42679, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [22ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: Network error: Failed to connect to peer master-127.8.104.190:42679(127.8.104.190:42679): Connection refused: /127.8.104.190:42679, [23ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: OK, deferred=Deferred@706494569(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/585998101@dab23fb, errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/978413659@4515aaf0)) after lookup
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.744661 8733 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:41038:
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.748283 8824 raft_consensus.cc:1273] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 2 FOLLOWER]: Refusing update from remote peer 89ecc82c1296466abc4576789e40284d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.749028 9424 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
19:59:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:24.749758 8733 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
19:59:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.750423 9419 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 89ecc82c1296466abc4576789e40284d. Latest consensus state: current_term: 2 leader_uuid: "89ecc82c1296466abc4576789e40284d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.750674 9419 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: This master's current role is: FOLLOWER
19:59:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.752179 9418 sys_catalog.cc:455] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "89ecc82c1296466abc4576789e40284d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.752476 9418 sys_catalog.cc:458] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d [sys.catalog]: This master's current role is: LEADER
19:59:24.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.754490 9419 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "89ecc82c1296466abc4576789e40284d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:24.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:24.754688 9419 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: This master's current role is: FOLLOWER
19:59:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:24.788962 9202 heartbeater.cc:646] Failed to heartbeat to 127.8.104.190:42679 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111)
19:59:25.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:25.250607 8718 consensus_peers.cc:489] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d -> Peer e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679): Couldn't send request to peer e11e01529b2f4bf88c980fdc7cc21f92. Status: Network error: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
19:59:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:25.732690 9063 heartbeater.cc:499] Master 127.8.104.189:34439 was elected leader, sending a full tablet report...
19:59:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:25.766119 8718 consensus_peers.cc:489] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d -> Peer e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679): Couldn't send request to peer e11e01529b2f4bf88c980fdc7cc21f92. Status: Network error: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
19:59:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:25.788856 9204 heartbeater.cc:499] Master 127.8.104.189:34439 was elected leader, sending a full tablet report...
19:59:26.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.204203 9346 heartbeater.cc:499] Master 127.8.104.189:34439 was elected leader, sending a full tablet report...
19:59:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.277259 8718 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111) [suppressed 21 similar messages]
19:59:26.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.278259 8718 consensus_peers.cc:489] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d -> Peer e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679): Couldn't send request to peer e11e01529b2f4bf88c980fdc7cc21f92. Status: Network error: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
19:59:26.341 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet e00e833ec2d443a68460764399105faf for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
19:59:26.341 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet b95b19d800b8486ab36c1352b8e6ba74 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
19:59:26.341 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=e00e833ec2d443a68460764399105faf@[60624d7b4dcf468da997ac3454bb427a(127.8.104.129:42643)[L],88f4731850ec489b8696c9a74004c187(127.8.104.130:34681),92e16bf1ab8c41e09fb4057ecffe50a1(127.8.104.131:42283)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=b95b19d800b8486ab36c1352b8e6ba74@[60624d7b4dcf468da997ac3454bb427a(127.8.104.129:42643),88f4731850ec489b8696c9a74004c187(127.8.104.130:34681)[L],92e16bf1ab8c41e09fb4057ecffe50a1(127.8.104.131:42283)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
19:59:26.342 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.8.104.189:34439, [1ms] Sub RPC GetTableLocations: received response from server master-127.8.104.189:34439: OK, deferred=null) after lookup
19:59:26.342 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 60624d7b4dcf468da997ac3454bb427a(127.8.104.129:42643)] connecting to peer
19:59:26.343 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 60624d7b4dcf468da997ac3454bb427a(127.8.104.129:42643)] Successfully connected to peer
19:59:26.356 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xb63d5a48, L:/127.0.0.1:52652 - R:/127.8.104.129:42643] using TOKEN/null
19:59:26.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.360642 8972 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.8.104.190:42679)
19:59:26.413 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.104.190:42679
19:59:26.413 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
19:59:26.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.414460 8610 external_mini_cluster.cc:1366] Running /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3Meco8/build/asan/bin/kudu
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/logs
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data/info.pb
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.190:42679
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.190
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46569
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36827/jwks.json
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
19:59:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
19:59:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.723515 8781 master_runner.cc:186] Error getting master registration for 127.8.104.190:42679: Network error: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111),
19:59:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.723719 8781 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:59:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.724262 9468 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:59:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.724575 9468 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:59:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.724692 9468 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:59:26.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.732429 9468 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:59:26.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.732549 9468 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
19:59:26.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.732599 9468 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:59:26.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.732637 9468 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:59:26.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.732688 9468 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:59:26.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.732726 9468 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:59:26.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.732770 9468 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36827/jwks.json
19:59:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.742643 9468 master_runner.cc:386] Master server non-default flags:
19:59:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:59:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.104.148:45579
19:59:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:59:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.104.190:42679,127.8.104.189:34439,127.8.104.188:43579
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.104.190:42679
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36827/jwks.json
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data/info.pb
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.104.190
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46569
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/logs
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 19:43:25 UTC on 24a791456cd2
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7836
19:59:26.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
19:59:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.744222 9468 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:59:26.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.745868 9468 file_cache.cc:492] Constructed file cache file cache with capacity 419430
19:59:26.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.747958 8718 consensus_peers.cc:489] T 00000000000000000000000000000000 P 89ecc82c1296466abc4576789e40284d -> Peer e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679): Couldn't send request to peer e11e01529b2f4bf88c980fdc7cc21f92. Status: Network error: Client connection negotiation failed: client connection to 127.8.104.190:42679: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
19:59:26.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.754060 9476 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:26.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.754138 9475 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:26.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.755147 9478 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:59:26.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.755632 9468 server_base.cc:1047] running on GCE node
19:59:26.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.756364 9468 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:59:26.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.757718 9468 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:59:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.758970 9468 hybrid_clock.cc:648] HybridClock initialized: now 1756670366758897 us; error 51 us; skew 500 ppm
19:59:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.759305 9468 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:59:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.761369 9468 webserver.cc:480] Webserver started at http://127.8.104.190:46569/ using document root <none> and password file <none>
19:59:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.762104 9468 fs_manager.cc:362] Metadata directory not provided
19:59:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.762236 9468 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:59:26.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.767359 9468 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.002s sys 0.003s
19:59:26.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.770071 9484 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:59:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.771377 9468 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.771538 9468 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data,/tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e11e01529b2f4bf88c980fdc7cc21f92"
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 19:59:20 on dist-test-slave-7hw0"
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.772240 9468 fs_report.cc:389] FS layout report
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/wal
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data/data
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:59:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:59:26.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.795895 9468 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
19:59:26.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.796646 9468 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:59:26.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.796856 9468 kserver.cc:163] Server-wide thread pool size limit: 3276
19:59:26.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.816989 9468 rpc_server.cc:307] RPC server started. Bound to: 127.8.104.190:42679
19:59:26.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.817008 9546 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.104.190:42679 every 8 connection(s)
19:59:26.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.818594 9468 server_base.cc:1179] Dumped server information to /tmp/dist-test-task3Meco8/test-tmp/mini-kudu-cluster2490395182307148644/master-0/data/info.pb
19:59:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.824792 9547 sys_catalog.cc:263] Verifying existing consensus state
19:59:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.827316 8610 external_mini_cluster.cc:1428] Started /tmp/dist-test-task3Meco8/build/asan/bin/kudu as pid 9468
19:59:26.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:59:26.827 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.104.189:34439
19:59:26.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
19:59:26.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.827857 9547 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: Bootstrap starting.
19:59:26.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.828109 8610 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task3Meco8/build/asan/bin/kudu pid 8699
19:59:26.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.829180 8773 generic_service.cc:196] Checking for leaks (request via RPC)
19:59:26.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.847404 9547 log.cc:826] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: Log is configured to *not* fsync() on all Append() calls
19:59:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.864636 9547 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:59:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.865231 9547 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: Bootstrap complete.
19:59:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.874601 9547 raft_consensus.cc:357] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.875550 9547 raft_consensus.cc:738] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: e11e01529b2f4bf88c980fdc7cc21f92, State: Initialized, Role: FOLLOWER
19:59:26.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.876325 9547 consensus_queue.cc:260] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:26.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.877265 9551 sys_catalog.cc:455] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:26.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.877657 9551 sys_catalog.cc:458] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: This master's current role is: FOLLOWER
19:59:26.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.878371 9547 sys_catalog.cc:564] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: configured and running, proceeding with master startup.
19:59:26.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.895714 9562 catalog_manager.cc:1261] Loaded cluster ID: a1178ae452f34170ababc7c815199e6f
19:59:26.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.895818 9562 catalog_manager.cc:1554] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: loading cluster ID for follower catalog manager: success
19:59:26.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.904263 9562 catalog_manager.cc:1576] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: acquiring CA information for follower catalog manager: success
19:59:26.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.907361 9562 catalog_manager.cc:1604] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:59:26.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.973322 8610 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task3Meco8/build/asan/bin/kudu with pid 8699
19:59:26.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:59:26.994 [ERROR - kudu-nio-0] (Connection.java:452) [peer master-127.8.104.189:34439(127.8.104.189:34439)] unexpected exception from downstream on [id: 0x61d09adf, L:/127.0.0.1:41032 - R:/127.8.104.189:34439]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) ~[?:1.8.0_275]
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:192)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379)
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255)
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132)
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
19:59:26.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.993829 8800 connection.cc:537] server connection from 127.0.0.1:54926 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
19:59:26.996 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state READY due to: [peer master-127.8.104.189:34439(127.8.104.189:34439)] unexpected exception from downstream on [id: 0x61d09adf, L:/127.0.0.1:41032 - R:/127.8.104.189:34439]
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.995832 9564 negotiation.cc:337] Failed RPC negotiation. Trace:
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 19:59:26.925830 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.8.104.189:34439 (local address 127.0.0.1:54834)
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 19:59:26.926016 (+ 186us) negotiation.cc:107] Waiting for socket to connect
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 19:59:26.926033 (+ 17us) client_negotiation.cc:174] Beginning negotiation
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 19:59:26.926162 (+ 129us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 19:59:26.994545 (+ 68383us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":66}
19:59:26.996 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.8.104.189:34439(127.8.104.189:34439) for tablet Kudu Master: [peer master-127.8.104.189:34439(127.8.104.189:34439)] unexpected exception from downstream on [id: 0x61d09adf, L:/127.0.0.1:41032 - R:/127.8.104.189:34439]
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:26.995896 9565 negotiation.cc:337] Failed RPC negotiation. Trace:
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 19:59:26.979444 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40058 (local address 127.8.104.190:42679)
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 19:59:26.979682 (+ 238us) server_negotiation.cc:206] Beginning negotiation
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 19:59:26.979693 (+ 11us) server_negotiation.cc:399] Waiting for connection header
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 19:59:26.995559 (+ 15866us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40058: BlockingRecv error: recv got EOF from 127.0.0.1:40058 (error 108)
19:59:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":100}
19:59:26.998 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.104.189:34439(127.8.104.189:34439)] handling channelInactive
19:59:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:26.996169 9468 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
19:59:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.001616 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:59:27.001 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.190:42679(127.8.104.190:42679)] connecting to peer
19:59:27.002 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.003 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.104.190:42679(127.8.104.190:42679)] Successfully connected to peer
19:59:27.004 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.004 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.004 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.018 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xe4bd9931, L:/127.0.0.1:40066 - R:/127.8.104.190:42679] using TOKEN/null
19:59:27.020 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:27.042 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.043 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.043 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.043 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.045 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:27.062 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.063 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.063 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.063 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.063 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:27.082 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.083 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.083 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.083 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.083 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:27.084 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] sending RPC to server master-127.8.104.189:34439, [155ms] received response from server master-127.8.104.189:34439: Network error: [peer master-127.8.104.189:34439(127.8.104.189:34439)] unexpected exception from downstream on [id: 0x61d09adf, L:/127.0.0.1:41032 - R:/127.8.104.189:34439], [155ms] delaying RPC due to: Network error: [peer master-127.8.104.189:34439(127.8.104.189:34439)] unexpected exception from downstream on [id: 0x61d09adf, L:/127.0.0.1:41032 - R:/127.8.104.189:34439], [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [163ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [163ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [178ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [179ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [202ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [202ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [203ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [222ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [222ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [242ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [242ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, deferred=Deferred@1665333282(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/585998101@24a135e2 -> (continuation of Deferred@311865773 after retry RPC after error@17426622) -> (continuation of Deferred@1979614037 after retry RPC after error@1490921005) -> (continuation of Deferred@2145480514 after retry RPC after error@836973592), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/978413659@3086fd5a -> (continuation of Deferred@311865773 after retry RPC after error@17426622) -> (continuation of Deferred@1979614037 after retry RPC after error@1490921005) -> (continuation of Deferred@2145480514 after retry RPC after error@836973592)))
19:59:27.085 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] sending RPC to server master-127.8.104.189:34439, [155ms] received response from server master-127.8.104.189:34439: Network error: [peer master-127.8.104.189:34439(127.8.104.189:34439)] unexpected exception from downstream on [id: 0x61d09adf, L:/127.0.0.1:41032 - R:/127.8.104.189:34439], [155ms] delaying RPC due to: Network error: [peer master-127.8.104.189:34439(127.8.104.189:34439)] unexpected exception from downstream on [id: 0x61d09adf, L:/127.0.0.1:41032 - R:/127.8.104.189:34439], [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [163ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [163ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [178ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [179ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [202ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [202ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [203ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [222ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [222ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [242ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [242ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, deferred=Deferred@1665333282(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/585998101@24a135e2 -> (continuation of Deferred@311865773 after retry RPC after error@17426622) -> (continuation of Deferred@1979614037 after retry RPC after error@1490921005) -> (continuation of Deferred@2145480514 after retry RPC after error@836973592), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/978413659@3086fd5a -> (continuation of Deferred@311865773 after retry RPC after error@17426622) -> (continuation of Deferred@1979614037 after retry RPC after error@1490921005) -> (continuation of Deferred@2145480514 after retry RPC after error@836973592)))
19:59:27.102 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.103 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.103 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.103 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.104 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:27.121 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.123 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.123 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.123 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.123 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:27.141 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.142 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.142 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.143 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.144 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:27.161 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.162 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.163 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.163 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.163 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:27.181 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.182 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.183 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.183 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.183 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:27.222 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.223 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.223 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.223 [INFO - pool-40-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.224 [WARN - pool-40-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.225458 9551 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:59:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.225800 9551 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:27.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.227615 9551 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:27.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:27.228856 9486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:27.242154 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:27.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.247969 8824 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" is_pre_election: true
19:59:27.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.248716 9488 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.249125 9551 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
19:59:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.274271 9571 raft_consensus.cc:491] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 89ecc82c1296466abc4576789e40284d)
19:59:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.274442 9571 raft_consensus.cc:513] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:27.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.274991 9571 leader_election.cc:290] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679), 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439)
19:59:27.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:27.277609 8799 leader_election.cc:336] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:27.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.281162 9512 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" is_pre_election: true
19:59:27.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.281617 9512 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa8c3788cc4a40c5bb7b2497e6bab159 in term 1.
19:59:27.282 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.282 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.282250 8799 leader_election.cc:304] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92, fa8c3788cc4a40c5bb7b2497e6bab159; no voters: 89ecc82c1296466abc4576789e40284d
19:59:27.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.282673 9571 raft_consensus.cc:2802] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 2 FOLLOWER]: Leader pre-election won for term 3
19:59:27.283 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.282806 9571 raft_consensus.cc:491] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 89ecc82c1296466abc4576789e40284d)
19:59:27.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.282910 9571 raft_consensus.cc:3058] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 2 FOLLOWER]: Advancing to term 3
19:59:27.283 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.283 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.284730 9571 raft_consensus.cc:513] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.285149 9571 leader_election.cc:290] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [CANDIDATE]: Term 3 election: Requested vote from peers e11e01529b2f4bf88c980fdc7cc21f92 (127.8.104.190:42679), 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439)
19:59:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.285612 9512 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "e11e01529b2f4bf88c980fdc7cc21f92"
19:59:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.285763 9512 raft_consensus.cc:3058] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 1 FOLLOWER]: Advancing to term 3
19:59:27.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:27.286358 8799 leader_election.cc:336] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.289029 9512 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa8c3788cc4a40c5bb7b2497e6bab159 in term 3.
19:59:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.289615 8799 leader_election.cc:304] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92, fa8c3788cc4a40c5bb7b2497e6bab159; no voters: 89ecc82c1296466abc4576789e40284d
19:59:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.289880 9571 raft_consensus.cc:2802] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 3 FOLLOWER]: Leader election won for term 3
19:59:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.290292 9571 raft_consensus.cc:695] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [term 3 LEADER]: Becoming Leader. State: Replica: fa8c3788cc4a40c5bb7b2497e6bab159, State: Running, Role: LEADER
19:59:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.290725 9571 consensus_queue.cc:237] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.292160 9575 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fa8c3788cc4a40c5bb7b2497e6bab159. Latest consensus state: current_term: 3 leader_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.292407 9575 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: This master's current role is: LEADER
19:59:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.293064 9577 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
19:59:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.295394 9577 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab]
19:59:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.295862 9577 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=d80f4524b53242f180d0127d58b6d72b]
19:59:27.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.297264 9577 tablet_loader.cc:96] loaded metadata for tablet 22590beba9194de0adb105319747e148 (table kudu_system.kudu_transactions [id=d80f4524b53242f180d0127d58b6d72b])
19:59:27.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.297689 9577 tablet_loader.cc:96] loaded metadata for tablet b95b19d800b8486ab36c1352b8e6ba74 (table txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab])
19:59:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.297972 9577 tablet_loader.cc:96] loaded metadata for tablet e00e833ec2d443a68460764399105faf (table txn_manager_fallback_rolling [id=3c83f2cf970e4d34aa8a7838bb1701ab])
19:59:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.298175 9577 catalog_manager.cc:1486] Initializing Kudu cluster ID...
19:59:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.298849 9577 catalog_manager.cc:1261] Loaded cluster ID: a1178ae452f34170ababc7c815199e6f
19:59:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.298923 9577 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
19:59:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.299888 9577 catalog_manager.cc:1506] Loading token signing keys...
19:59:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.300596 9577 catalog_manager.cc:5966] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159: Loaded TSK: 0
19:59:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.301313 9577 catalog_manager.cc:1516] Initializing in-progress tserver states...
19:59:27.342 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:27.343 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.343 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.343 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:27.346 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
19:59:27.347 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=245), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [3ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [22ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [22ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [42ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [42ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [62ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [62ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [82ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [82ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [122ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [123ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [182ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [182ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [242ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [244ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, deferred=Deferred@829270612(state=PENDING, result=null, callback=(continuation of Deferred@332889237 after retry RPC after error@688840872) -> (continuation of Deferred@1674655615 after retry RPC after error@1068633691) -> (continuation of Deferred@388624108 after retry RPC after error@1232885505) -> (continuation of Deferred@603777950 after retry RPC after error@211024236) -> (continuation of Deferred@1410676892 after retry RPC after error@2147133760) -> (continuation of Deferred@1000217914 after retry RPC after error@685099964) -> (continuation of Deferred@1442437090 after retry RPC after error@2145633460), errback=(continuation of Deferred@332889237 after retry RPC after error@688840872) -> (continuation of Deferred@1674655615 after retry RPC after error@1068633691) -> (continuation of Deferred@388624108 after retry RPC after error@1232885505) -> (continuation of Deferred@603777950 after retry RPC after error@211024236) -> (continuation of Deferred@1410676892 after retry RPC after error@2147133760) -> (continuation of Deferred@1000217914 after retry RPC after error@685099964) -> (continuation of Deferred@1442437090 after retry RPC after error@2145633460))) after lookup
19:59:27.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.356369 9344 heartbeater.cc:344] Connected to a master server at 127.8.104.190:42679
19:59:27.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.359288 9502 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "92e16bf1ab8c41e09fb4057ecffe50a1" instance_seqno: 1756670363330275) as {username='slave'} at 127.8.104.131:38145; Asking this server to re-register.
19:59:27.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.360348 9344 heartbeater.cc:461] Registering TS with master...
19:59:27.362 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [3ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [22ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [22ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [42ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [42ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [62ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [62ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [82ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [82ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [122ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [123ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [182ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [182ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [242ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [244ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: OK, [246ms] sending RPC to server master-127.8.104.188:43579, [251ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, deferred=Deferred@829270612(state=PENDING, result=null, callback=(continuation of Deferred@332889237 after retry RPC after error@688840872) -> (continuation of Deferred@1674655615 after retry RPC after error@1068633691) -> (continuation of Deferred@388624108 after retry RPC after error@1232885505) -> (continuation of Deferred@603777950 after retry RPC after error@211024236) -> (continuation of Deferred@1410676892 after retry RPC after error@2147133760) -> (continuation of Deferred@1000217914 after retry RPC after error@685099964) -> (continuation of Deferred@1442437090 after retry RPC after error@2145633460) -> (continuation of Deferred@340047187 after retry RPC@1546710705), errback=(continuation of Deferred@332889237 after retry RPC after error@688840872) -> (continuation of Deferred@1674655615 after retry RPC after error@1068633691) -> (continuation of Deferred@388624108 after retry RPC after error@1232885505) -> (continuation of Deferred@603777950 after retry RPC after error@211024236) -> (continuation of Deferred@1410676892 after retry RPC after error@2147133760) -> (continuation of Deferred@1000217914 after retry RPC after error@685099964) -> (continuation of Deferred@1442437090 after retry RPC after error@2145633460) -> (continuation of Deferred@340047187 after retry RPC@1546710705)))
19:59:27.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.366451 9502 ts_manager.cc:194] Registered new tserver with Master: 92e16bf1ab8c41e09fb4057ecffe50a1 (127.8.104.131:42283)
19:59:27.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.368103 8944 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.017s from last keepalive heartbeat (effective timeout is 1.000s)
19:59:27.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:27.369376 8972 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.8.104.189:34439)
19:59:27.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:27.377900 8844 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:54940 (request call id 14) took 29 ms (client timeout 4 ms). Trace:
19:59:27.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:27.378207 8844 rpcz_store.cc:269] 0831 19:59:27.348065 (+ 0us) service_pool.cc:168] Inserting onto call queue
19:59:27.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 19:59:27.349708 (+ 1643us) service_pool.cc:225] Handling call
19:59:27.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 19:59:27.377809 (+ 28101us) inbound_call.cc:173] Queueing success response
19:59:27.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":9518,"thread_start_us":12629,"threads_started":4}
19:59:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:27.381984 9578 tablet_replica.cc:1304] Aborted: operation has been aborted: cancelling pending write operations
19:59:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28: runtime error: signed integer overflow: 602678013178 + 9223371434176764036 cannot be represented in type 'long'
19:59:27.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.414156 9512 raft_consensus.cc:1273] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Refusing update from remote peer fa8c3788cc4a40c5bb7b2497e6bab159: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
19:59:27.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:27.418505 8799 consensus_peers.cc:489] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 -> Peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Couldn't send request to peer 89ecc82c1296466abc4576789e40284d. Status: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
19:59:27.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.419075 9571 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
19:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.431223 8813 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40208:
19:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
19:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
19:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
19:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
19:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
19:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
19:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
19:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
19:59:27.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:27.437590 8813 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
19:59:27.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.453724 9551 sys_catalog.cc:455] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fa8c3788cc4a40c5bb7b2497e6bab159. Latest consensus state: current_term: 3 leader_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:27.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.454067 9551 sys_catalog.cc:458] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: This master's current role is: FOLLOWER
19:59:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.457801 9601 sys_catalog.cc:455] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.458118 9601 sys_catalog.cc:458] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: This master's current role is: FOLLOWER
19:59:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.458339 9601 sys_catalog.cc:455] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.458488 9601 sys_catalog.cc:458] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [sys.catalog]: This master's current role is: FOLLOWER
19:59:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.464332 9571 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } } }
19:59:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.464635 9571 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 [sys.catalog]: This master's current role is: LEADER
19:59:27.473 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 60624d7b4dcf468da997ac3454bb427a (127.8.104.129:42643): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1756670364 last_transition_timestamp: 1756670367
19:59:27.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.731540 9061 heartbeater.cc:344] Connected to a master server at 127.8.104.190:42679
19:59:27.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.733711 9502 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "60624d7b4dcf468da997ac3454bb427a" instance_seqno: 1756670362400135) as {username='slave'} at 127.8.104.129:33929; Asking this server to re-register.
19:59:27.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.734313 9061 heartbeater.cc:461] Registering TS with master...
19:59:27.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.735467 9502 ts_manager.cc:194] Registered new tserver with Master: 60624d7b4dcf468da997ac3454bb427a (127.8.104.129:42643)
19:59:27.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.800837 9202 heartbeater.cc:344] Connected to a master server at 127.8.104.190:42679
19:59:27.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.802314 9502 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "88f4731850ec489b8696c9a74004c187" instance_seqno: 1756670362840543) as {username='slave'} at 127.8.104.130:49171; Asking this server to re-register.
19:59:27.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.803100 9202 heartbeater.cc:461] Registering TS with master...
19:59:27.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:27.804569 9502 ts_manager.cc:194] Registered new tserver with Master: 88f4731850ec489b8696c9a74004c187 (127.8.104.130:34681)
19:59:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:27.890336 8799 consensus_peers.cc:489] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 -> Peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Couldn't send request to peer 89ecc82c1296466abc4576789e40284d. Status: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
19:59:28.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7f226adb6d38 in kudu::operator+(kudu::MonoTime const&, kudu::MonoDelta const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28
19:59:28.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:28.437120 8799 consensus_peers.cc:489] T 00000000000000000000000000000000 P fa8c3788cc4a40c5bb7b2497e6bab159 -> Peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Couldn't send request to peer 89ecc82c1296466abc4576789e40284d. Status: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
19:59:28.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0xa66af5 in kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/countdown_latch.h:99:38
19:59:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7f227e590af2 in kudu::Promise<kudu::Status>::WaitFor(kudu::MonoDelta const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/promise.h:57:16
19:59:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7f227e580604 in kudu::master::Master::WaitForTxnManagerInit(kudu::MonoTime) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/master.cc:482:48
19:59:28.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7f227e6e4c7e in kudu::transactions::TxnManager::CheckInitialized(kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:283:21
19:59:28.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7f227e6e585b in kudu::transactions::TxnManager::KeepTransactionAlive(long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager.cc:226:3
19:59:28.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7f227e6ea49f in kudu::transactions::TxnManagerServiceImpl::KeepTransactionAlive(kudu::transactions::KeepTransactionAliveRequestPB const*, kudu::transactions::KeepTransactionAliveResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/master/txn_manager_service.cc:159:42
19:59:28.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7f227860f2d6 in kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/asan/src/kudu/master/txn_manager.service.cc:271:13
19:59:28.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7f227860f005 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
19:59:28.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7f2271c092ce in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/7.5.0/bits/std_function.h:706:14
19:59:28.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7f2271c081ae in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3
19:59:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7f2271c0e3e5 in kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:229:15
19:59:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7f2271c0ed0f in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:92:5
19:59:28.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7f2271c0eb38 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/7.5.0/bits/std_function.h:316:2
19:59:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0xa774dc in std::function<void ()>::operator()() const ../../../include/c++/7.5.0/bits/std_function.h:706:14
19:59:28.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7f226ae80cc8 in kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:694:3
19:59:28.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7f226e2a26da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
19:59:28.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7f2266fd471e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12171e)
19:59:28.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:59:28.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/monotime.cc:332:28 in
19:59:28.644 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.8.104.188:43579(127.8.104.188:43579)] handling channelInactive
19:59:28.644 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state READY due to: connection closed
19:59:28.645 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.8.104.188:43579(127.8.104.188:43579) for tablet Kudu Master: connection closed
19:59:28.663 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
19:59:28.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:28.730086 9062 heartbeater.cc:499] Master 127.8.104.188:43579 was elected leader, sending a full tablet report...
19:59:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:28.798336 9203 heartbeater.cc:499] Master 127.8.104.188:43579 was elected leader, sending a full tablet report...
19:59:28.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:28.899376 9615 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
19:59:28.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:28.899562 9615 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:28.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:28.900281 9615 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:28.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:28.902330 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:28.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:28.902518 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:28.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:28.902647 9488 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:28.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:28.902850 9615 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:28.995 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:28.996 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:28.997 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:28.997 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:28.997 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:28.997 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:28.997 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:28.997 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:28.997 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:29.002 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:29.002 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:29.002 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.002 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.002 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.002 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.003 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.003 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.003 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:29.022 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:29.022 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:29.022 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.023 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.023 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.023 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.023 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.023 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.024 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:29.042 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:29.042 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:29.043 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.043 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.043 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.044 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.044 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.044 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.044 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:29.061 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:29.062 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:29.062 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.062 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.063 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.063 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.063 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.063 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.063 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:29.082 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:29.082 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:29.083 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.083 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.083 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.083 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.083 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.083 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.084 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:29.161 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:29.162 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:29.162 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.162 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.162 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.163 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.163 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.163 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.163 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:29.202 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:29.202 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:29.203 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.203 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.203 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.203 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.204 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.204 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.204 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:29.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:29.210397 9345 heartbeater.cc:499] Master 127.8.104.188:43579 was elected leader, sending a full tablet report...
19:59:29.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:29.249673 9615 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
19:59:29.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:29.249886 9615 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:29.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:29.250671 9615 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:29.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:29.252107 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:29.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:29.252295 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:29.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:29.252439 9488 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:29.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:29.252736 9615 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:29.322 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:29.322 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:29.323 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.323 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.323 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.323 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.323 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.323 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.324 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:29.842 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:29.842 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:29.842 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.843 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.843 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:29.843 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.843 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.843 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:29.843 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:29.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:29.903775 9625 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
19:59:29.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:29.904001 9625 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:29.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:29.904688 9625 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:29.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:29.906728 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:29.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:29.906878 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:29.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:29.907004 9486 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:29.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:29.907234 9625 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:30.514431 9631 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
19:59:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:30.514603 9631 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:30.515167 9631 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:30.517181 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:30.517386 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:30.517508 9486 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:30.517751 9631 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:30.821 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:30.822 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:30.822 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:30.822 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:30.822 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:30.823 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:30.823 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:30.823 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:30.823 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:31.226136 9641 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
19:59:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:31.226313 9641 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:31.226876 9641 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:31.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:31.228585 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:31.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:31.228760 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:31.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:31.228899 9486 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:31.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:31.229153 9641 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:32.004495 9468 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111),
19:59:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:32.004667 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:59:32.562 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:32.562 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:32.563 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:32.563 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:32.563 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:32.563 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:32.563 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:32.563 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:32.564 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:32.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:32.899550 9656 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
19:59:32.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:32.899739 9656 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:32.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:32.900364 9656 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:32.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:32.901352 9486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111) [suppressed 10 similar messages]
19:59:32.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:32.902388 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:32.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:32.902536 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:32.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:32.902700 9486 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:32.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:32.902967 9656 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:33.461 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:33.462 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:33.462 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:33.462 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:33.462 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:33.462 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:33.462 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:33.463 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:33.463 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:34.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:34.338263 9668 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
19:59:34.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:34.338428 9668 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:34.339015 9668 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:34.342550 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:34.342864 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:34.343037 9488 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:34.343292 9668 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:35.442 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:35.442 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:35.443 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:35.443 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:35.443 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:35.443 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:35.443 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:35.443 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:35.444 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:37.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:37.007722 9468 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111),
19:59:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:37.007886 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:59:37.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:37.231659 9692 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
19:59:37.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:37.231877 9692 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:37.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:37.232426 9692 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:37.234148 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:37.234318 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:37.234436 9488 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:37.234659 9692 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:38.661 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:38.662 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:38.662 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:38.662 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:38.662 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:38.663 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:38.663 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:38.663 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:38.663 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:39.622 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:39.622 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:39.623 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:39.623 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:39.623 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:39.623 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:39.623 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:39.624 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:39.624 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:39.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:39.940246 9711 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
19:59:39.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:39.940448 9711 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:39.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:39.941099 9711 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:39.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:39.942102 9486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111) [suppressed 5 similar messages]
19:59:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:39.943372 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:39.943522 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:39.943701 9488 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:39.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:39.943945 9711 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:42.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:42.010718 9468 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111),
19:59:42.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:42.010867 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:59:43.121 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:43.122 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:43.122 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:43.122 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:43.122 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:43.122 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:43.123 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:43.123 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:43.123 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:43.282 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:43.282 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:43.283 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:43.283 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:43.283 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:43.283 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:43.283 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:43.283 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:43.283 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:44.157408 9742 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
19:59:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:44.157624 9742 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:44.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:44.158146 9742 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:44.160089 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:44.160251 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:44.160416 9486 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:44.160684 9742 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:44.982 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:44.983 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:44.983 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:44.983 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:44.983 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:44.983 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:44.983 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:44.983 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:44.984 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:46.022 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:46.022 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:46.023 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:46.023 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:46.023 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:46.023 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:46.023 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:46.023 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:46.024 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:46.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:46.771895 9763 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
19:59:46.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:46.772080 9763 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
19:59:46.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:46.772683 9763 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
19:59:46.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:46.773396 9486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111) [suppressed 3 similar messages]
19:59:46.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:46.774530 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
19:59:46.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 19:59:46.774696 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
19:59:46.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:46.774799 9488 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
19:59:46.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:46.775005 9763 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
19:59:46.961 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:46.962 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:46.962 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:46.962 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:46.962 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:46.962 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:46.962 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:46.962 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:46.963 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:47.014341 9468 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111),
19:59:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:47.014504 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:59:48.862 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:48.862 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:48.862 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:48.863 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:48.863 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:48.863 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:48.863 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:48.863 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:48.864 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:51.502 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:51.502 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:51.503 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:51.503 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:51.503 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:51.503 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:51.503 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:51.503 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:51.504 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:52.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:52.017145 9468 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111),
19:59:52.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:52.017335 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:59:54.862 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:54.862 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:54.862 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:54.862 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:54.862 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:54.863 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:54.863 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:54.863 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:54.863 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:57.019942 9468 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111),
19:59:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 19:59:57.020113 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:59:58.462 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:58.462 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:58.463 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:58.463 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:58.463 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:58.463 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:58.463 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:58.463 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:58.464 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:58.822 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:58.823 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:58.823 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:58.823 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:58.823 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:58.823 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:58.823 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:58.824 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:58.824 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
19:59:59.502 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
19:59:59.502 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
19:59:59.502 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:59.503 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:59.503 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
19:59:59.503 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:59.503 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:59.503 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
19:59:59.503 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
20:00:00.661 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
20:00:00.662 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
20:00:00.662 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:00.662 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:00.662 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:00.662 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:00.663 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:00.663 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:00.663 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
20:00:00.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:00.892434 9862 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
20:00:00.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:00.892640 9862 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
20:00:00.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:00.893249 9862 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
20:00:00.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:00.894163 9486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111) [suppressed 1 similar messages]
20:00:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:00.895277 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
20:00:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:00.895440 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
20:00:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:00.895569 9486 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
20:00:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:00.895823 9862 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
20:00:02.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:02.022931 9468 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111),
20:00:02.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:02.023097 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:00:02.921 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
20:00:02.922 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
20:00:02.922 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:02.922 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:02.922 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:02.923 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:02.923 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:02.923 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:02.923 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
20:00:04.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:04.194895 9885 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
20:00:04.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:04.195070 9885 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
20:00:04.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:04.195621 9885 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
20:00:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:04.197780 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
20:00:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:04.198158 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
20:00:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:04.198267 9488 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
20:00:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:04.198479 9885 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
20:00:06.561 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
20:00:06.562 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
20:00:06.562 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:06.562 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:06.563 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:06.563 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:06.563 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:06.563 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:06.563 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
20:00:07.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:07.025769 9468 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111),
20:00:07.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:07.025947 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:00:07.181 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
20:00:07.182 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
20:00:07.182 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:07.182 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:07.182 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:07.182 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:07.182 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:07.183 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:07.183 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
20:00:07.842 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
20:00:07.842 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
20:00:07.843 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:07.843 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:07.843 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:07.843 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:07.843 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:07.843 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:07.844 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
20:00:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:07.929215 9916 raft_consensus.cc:491] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader fa8c3788cc4a40c5bb7b2497e6bab159)
20:00:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:07.929399 9916 raft_consensus.cc:513] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e11e01529b2f4bf88c980fdc7cc21f92" member_type: VOTER last_known_addr { host: "127.8.104.190" port: 42679 } } peers { permanent_uuid: "89ecc82c1296466abc4576789e40284d" member_type: VOTER last_known_addr { host: "127.8.104.189" port: 34439 } } peers { permanent_uuid: "fa8c3788cc4a40c5bb7b2497e6bab159" member_type: VOTER last_known_addr { host: "127.8.104.188" port: 43579 } }
20:00:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:07.930078 9916 leader_election.cc:290] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439), fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579)
20:00:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:07.930941 9486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:00:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:07.932057 9488 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fa8c3788cc4a40c5bb7b2497e6bab159 (127.8.104.188:43579): Network error: Client connection negotiation failed: client connection to 127.8.104.188:43579: connect: Connection refused (error 111)
20:00:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:07.932240 9486 leader_election.cc:336] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 89ecc82c1296466abc4576789e40284d (127.8.104.189:34439): Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111)
20:00:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:07.932360 9486 leader_election.cc:304] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e11e01529b2f4bf88c980fdc7cc21f92; no voters: 89ecc82c1296466abc4576789e40284d, fa8c3788cc4a40c5bb7b2497e6bab159
20:00:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:07.932543 9916 raft_consensus.cc:2747] T 00000000000000000000000000000000 P e11e01529b2f4bf88c980fdc7cc21f92 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
20:00:11.562 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
20:00:11.562 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
20:00:11.563 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:11.563 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:11.563 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:11.563 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:11.563 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:11.563 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:11.564 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
20:00:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:12.028635 9468 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111),
20:00:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:12.028807 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:00:13.342 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
20:00:13.343 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
20:00:13.343 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:13.343 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:13.343 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:13.343 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:13.343 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:13.343 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:13.344 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
20:00:15.721 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
20:00:15.722 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
20:00:15.722 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:15.722 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:15.722 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:15.722 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:15.722 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:15.722 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:15.723 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
20:00:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:17.031644 9468 master_runner.cc:186] Error getting master registration for 127.8.104.189:34439: Network error: Client connection negotiation failed: client connection to 127.8.104.189:34439: connect: Connection refused (error 111),
20:00:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:17.031810 9468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:00:17.602 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.188:43579(127.8.104.188:43579)] connecting to peer
20:00:17.602 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.8.104.189:34439(127.8.104.189:34439)] connecting to peer
20:00:17.603 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:17.603 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.104.188:43579(127.8.104.188:43579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:17.603 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:17.603 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.189:34439(127.8.104.189:34439)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:17.603 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.188:43579: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579
20:00:17.603 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.104.189:34439: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439
20:00:17.603 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439; will retry
20:00:17.604 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=36, TimeoutTracker(timeout=50000, elapsed=48608), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [3ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [6ms] refreshing cache from master, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [7ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [8ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [8ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [8ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [26ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [28ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [28ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [29ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [29ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [46ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [48ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [48ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [49ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [66ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [68ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [68ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [68ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [69ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [86ms] refreshing cache from master, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [88ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [88ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [88ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [89ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [166ms] refreshing cache from master, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [167ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [168ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [168ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [168ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [206ms] refreshing cache from master, [206ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [208ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [208ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [209ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [209ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [326ms] refreshing cache from master, [327ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [327ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [327ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [328ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [328ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [329ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [329ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [846ms] refreshing cache from master, [846ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [846ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [848ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [848ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [848ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [848ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [1826ms] refreshing cache from master, [1826ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [1826ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [1827ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [1827ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [1828ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [1828ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [1828ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [3566ms] refreshing cache from master, [3567ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [3567ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [3567ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [3568ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [3568ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [3569ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [3569ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [4466ms] refreshing cache from master, [4466ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [4466ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [4467ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [4467ms] too many traces: truncated at 100 traces, deferred=Deferred@1302957249(state=PENDING, result=null, callback=(continuation of Deferred@581007372 after retry RPC after error@1759754833) -> (continuation of Deferred@1607695714 after retry RPC after error@1578970589) -> (continuation of Deferred@2002079498 after retry RPC after error@43078275) -> (continuation of Deferred@1189939166 after retry RPC after error@1906948379) -> (continuation of Deferred@752452923 after retry RPC after error@609537897) -> (continuation of Deferred@178913226 after retry RPC after error@922021605) -> (continuation of Deferred@1457804637 after retry RPC after error@1422522223) -> (continuation of Deferred@291177875 after retry RPC after error@13193883) -> (continuation of Deferred@1962127714 after retry RPC after error@1872384540) -> (continuation of Deferred@648324577 after retry RPC after error@1815307812) -> (continuation of Deferred@86159969 after retry RPC after error@940903019) -> (continuation of Deferred@991767333 after retry RPC after error@1290303524) -> (continuation of Deferred@1548319330 after retry RPC after error@1285514706) -> (continuation of Deferred@512310119 after retry RPC after error@296704458) -> (continuation of Deferred@102939639 after retry RPC after error@832892692) -> (continuation of Deferred@1713481796 after retry RPC after error@229362755) -> (continuation of Deferred@516236034 after retry RPC after error@143635329) -> (continuation of Deferred@1669810524 after retry RPC after error@815614139) -> (continuation of Deferred@125280409 after retry RPC after error@1922107469) -> (continuation of Deferred@730165994 after retry RPC after error@1904689073) -> (continuation of Deferred@1263417968 after retry RPC after error@837255220) -> (continuation of Deferred@838186688 after retry RPC after error@2034728440) -> (continuation of Deferred@1138771226 after retry RPC after error@1309383538) -> (continuation of Deferred@1389656005 after retry RPC after error@280914138) -> (continuation of Deferred@1370957634 after retry RPC after error@1140192781) -> (continuation of Deferred@846126057 after retry RPC after error@1375051206) -> (continuation of Deferred@965750157 after retry RPC after error@1438245164) -> (continuation of Deferred@290249185 after retry RPC after error@1823112611) -> (continuation of Deferred@1416546814 after retry RPC after error@1321530078) -> (continuation of Deferred@967878966 after retry RPC after error@113810630) -> (continuation of Deferred@617321688 after retry RPC after error@1307627677) -> (continuation of Deferred@1899309324 after retry RPC after error@869905728) -> (continuation of Deferred@1263307447 after retry RPC after error@7593851) -> (continuation of Deferred@1482095408 after retry RPC after error@651896527) -> (continuation of Deferred@158140817 after retry RPC after error@402364352), errback=(continuation of Deferred@581007372 after retry RPC after error@1759754833) -> (continuation of Deferred@1607695714 after retry RPC after error@1578970589) -> (continuation of Deferred@2002079498 after retry RPC after error@43078275) -> (continuation of Deferred@1189939166 after retry RPC after error@1906948379) -> (continuation of Deferred@752452923 after retry RPC after error@609537897) -> (continuation of Deferred@178913226 after retry RPC after error@922021605) -> (continuation of Deferred@1457804637 after retry RPC after error@1422522223) -> (continuation of Deferred@291177875 after retry RPC after error@13193883) -> (continuation of Deferred@1962127714 after retry RPC after error@1872384540) -> (continuation of Deferred@648324577 after retry RPC after error@1815307812) -> (continuation of Deferred@86159969 after retry RPC after error@940903019) -> (continuation of Deferred@991767333 after retry RPC after error@1290303524) -> (continuation of Deferred@1548319330 after retry RPC after error@1285514706) -> (continuation of Deferred@512310119 after retry RPC after error@296704458) -> (continuation of Deferred@102939639 after retry RPC after error@832892692) -> (continuation of Deferred@1713481796 after retry RPC after error@229362755) -> (continuation of Deferred@516236034 after retry RPC after error@143635329) -> (continuation of Deferred@1669810524 after retry RPC after error@815614139) -> (continuation of Deferred@125280409 after retry RPC after error@1922107469) -> (continuation of Deferred@730165994 after retry RPC after error@1904689073) -> (continuation of Deferred@1263417968 after retry RPC after error@837255220) -> (continuation of Deferred@838186688 after retry RPC after error@2034728440) -> (continuation of Deferred@1138771226 after retry RPC after error@1309383538) -> (continuation of Deferred@1389656005 after retry RPC after error@280914138) -> (continuation of Deferred@1370957634 after retry RPC after error@1140192781) -> (continuation of Deferred@846126057 after retry RPC after error@1375051206) -> (continuation of Deferred@965750157 after retry RPC after error@1438245164) -> (continuation of Deferred@290249185 after retry RPC after error@1823112611) -> (continuation of Deferred@1416546814 after retry RPC after error@1321530078) -> (continuation of Deferred@967878966 after retry RPC after error@113810630) -> (continuation of Deferred@617321688 after retry RPC after error@1307627677) -> (continuation of Deferred@1899309324 after retry RPC after error@869905728) -> (continuation of Deferred@1263307447 after retry RPC after error@7593851) -> (continuation of Deferred@1482095408 after retry RPC after error@651896527) -> (continuation of Deferred@158140817 after retry RPC after error@402364352)))
20:00:17.606 [DEBUG - main] (Connection.java:575) [peer master-127.8.104.190:42679(127.8.104.190:42679)] disconnecting while in state READY
20:00:17.606 [DEBUG - main] (Connection.java:575) [peer master-127.8.104.189:34439(127.8.104.189:34439)] disconnecting while in state TERMINATED
20:00:17.606 [DEBUG - main] (Connection.java:575) [peer 60624d7b4dcf468da997ac3454bb427a(127.8.104.129:42643)] disconnecting while in state READY
20:00:17.606 [DEBUG - main] (Connection.java:575) [peer master-127.8.104.188:43579(127.8.104.188:43579)] disconnecting while in state TERMINATED
20:00:17.607 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:00:17.607 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.104.190:42679(127.8.104.190:42679)] handling channelInactive
20:00:17.608 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.104.190:42679(127.8.104.190:42679)] cleaning up while in state READY due to: connection closed
20:00:17.608 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 60624d7b4dcf468da997ac3454bb427a(127.8.104.129:42643)] handling channelInactive
20:00:17.608 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 60624d7b4dcf468da997ac3454bb427a(127.8.104.129:42643)] cleaning up while in state READY due to: connection closed
20:00:17.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:17.608369 8610 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task3Meco8/build/asan/bin/kudu pid 8903
20:00:17.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:17.626230 9056 generic_service.cc:196] Checking for leaks (request via RPC)
20:00:17.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:17.752596 8610 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task3Meco8/build/asan/bin/kudu with pid 8903
20:00:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:17.774928 9081 connection.cc:537] server connection from 127.8.104.129:59167 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:00:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:17.775061 9083 connection.cc:537] client connection to 127.8.104.129:42643 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:00:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:17.775283 8610 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task3Meco8/build/asan/bin/kudu pid 9066
20:00:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:17.775255 9083 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:00:17.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:17.777055 9083 consensus_peers.cc:489] T b95b19d800b8486ab36c1352b8e6ba74 P 88f4731850ec489b8696c9a74004c187 -> Peer 60624d7b4dcf468da997ac3454bb427a (127.8.104.129:42643): Couldn't send request to peer 60624d7b4dcf468da997ac3454bb427a. Status: Network error: Client connection negotiation failed: client connection to 127.8.104.129:42643: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:00:17.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:17.781777 9197 generic_service.cc:196] Checking for leaks (request via RPC)
20:00:17.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:17.883067 8610 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task3Meco8/build/asan/bin/kudu with pid 9066
20:00:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 20:00:17.902603 9485 connection.cc:537] server connection from 127.8.104.130:49171 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:00:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:17.902848 8610 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task3Meco8/build/asan/bin/kudu pid 9207
20:00:17.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:17.909788 9339 generic_service.cc:196] Checking for leaks (request via RPC)
20:00:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:18.010689 8610 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task3Meco8/build/asan/bin/kudu with pid 9207
20:00:18.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:18.030107 8610 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task3Meco8/build/asan/bin/kudu pid 9468
20:00:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:18.036967 9542 generic_service.cc:196] Checking for leaks (request via RPC)
20:00:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 20:00:18.149492 8610 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task3Meco8/build/asan/bin/kudu with pid 9468
20:00:18.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T20:00:18Z chronyd exiting
20:00:18.305 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=36, TimeoutTracker(timeout=50000, elapsed=48608), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [3ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [6ms] refreshing cache from master, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [7ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [8ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [8ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [8ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [26ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [28ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [28ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [29ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [29ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [46ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [48ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [48ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [49ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [66ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [68ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [68ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [68ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [69ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [86ms] refreshing cache from master, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [88ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [88ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [88ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [89ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [166ms] refreshing cache from master, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [167ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [168ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [168ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [168ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [206ms] refreshing cache from master, [206ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [208ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [208ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [209ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [209ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [326ms] refreshing cache from master, [327ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [327ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [327ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [328ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [328ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [329ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [329ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [846ms] refreshing cache from master, [846ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [846ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [848ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [848ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [848ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [848ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [1826ms] refreshing cache from master, [1826ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [1826ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [1827ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [1827ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [1828ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [1828ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [1828ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [3566ms] refreshing cache from master, [3567ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [3567ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [3567ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [3568ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [3568ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [3569ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [3569ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [4466ms] refreshing cache from master, [4466ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [4466ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [4467ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [4467ms] too many traces: truncated at 100 traces, deferred=Deferred@1302957249(state=PENDING, result=null, callback=(continuation of Deferred@581007372 after retry RPC after error@1759754833) -> (continuation of Deferred@1607695714 after retry RPC after error@1578970589) -> (continuation of Deferred@2002079498 after retry RPC after error@43078275) -> (continuation of Deferred@1189939166 after retry RPC after error@1906948379) -> (continuation of Deferred@752452923 after retry RPC after error@609537897) -> (continuation of Deferred@178913226 after retry RPC after error@922021605) -> (continuation of Deferred@1457804637 after retry RPC after error@1422522223) -> (continuation of Deferred@291177875 after retry RPC after error@13193883) -> (continuation of Deferred@1962127714 after retry RPC after error@1872384540) -> (continuation of Deferred@648324577 after retry RPC after error@1815307812) -> (continuation of Deferred@86159969 after retry RPC after error@940903019) -> (continuation of Deferred@991767333 after retry RPC after error@1290303524) -> (continuation of Deferred@1548319330 after retry RPC after error@1285514706) -> (continuation of Deferred@512310119 after retry RPC after error@296704458) -> (continuation of Deferred@102939639 after retry RPC after error@832892692) -> (continuation of Deferred@1713481796 after retry RPC after error@229362755) -> (continuation of Deferred@516236034 after retry RPC after error@143635329) -> (continuation of Deferred@1669810524 after retry RPC after error@815614139) -> (continuation of Deferred@125280409 after retry RPC after error@1922107469) -> (continuation of Deferred@730165994 after retry RPC after error@1904689073) -> (continuation of Deferred@1263417968 after retry RPC after error@837255220) -> (continuation of Deferred@838186688 after retry RPC after error@2034728440) -> (continuation of Deferred@1138771226 after retry RPC after error@1309383538) -> (continuation of Deferred@1389656005 after retry RPC after error@280914138) -> (continuation of Deferred@1370957634 after retry RPC after error@1140192781) -> (continuation of Deferred@846126057 after retry RPC after error@1375051206) -> (continuation of Deferred@965750157 after retry RPC after error@1438245164) -> (continuation of Deferred@290249185 after retry RPC after error@1823112611) -> (continuation of Deferred@1416546814 after retry RPC after error@1321530078) -> (continuation of Deferred@967878966 after retry RPC after error@113810630) -> (continuation of Deferred@617321688 after retry RPC after error@1307627677) -> (continuation of Deferred@1899309324 after retry RPC after error@869905728) -> (continuation of Deferred@1263307447 after retry RPC after error@7593851) -> (continuation of Deferred@1482095408 after retry RPC after error@651896527) -> (continuation of Deferred@158140817 after retry RPC after error@402364352), errback=(continuation of Deferred@581007372 after retry RPC after error@1759754833) -> (continuation of Deferred@1607695714 after retry RPC after error@1578970589) -> (continuation of Deferred@2002079498 after retry RPC after error@43078275) -> (continuation of Deferred@1189939166 after retry RPC after error@1906948379) -> (continuation of Deferred@752452923 after retry RPC after error@609537897) -> (continuation of Deferred@178913226 after retry RPC after error@922021605) -> (continuation of Deferred@1457804637 after retry RPC after error@1422522223) -> (continuation of Deferred@291177875 after retry RPC after error@13193883) -> (continuation of Deferred@1962127714 after retry RPC after error@1872384540) -> (continuation of Deferred@648324577 after retry RPC after error@1815307812) -> (continuation of Deferred@86159969 after retry RPC after error@940903019) -> (continuation of Deferred@991767333 after retry RPC after error@1290303524) -> (continuation of Deferred@1548319330 after retry RPC after error@1285514706) -> (continuation of Deferred@512310119 after retry RPC after error@296704458) -> (continuation of Deferred@102939639 after retry RPC after error@832892692) -> (continuation of Deferred@1713481796 after retry RPC after error@229362755) -> (continuation of Deferred@516236034 after retry RPC after error@143635329) -> (continuation of Deferred@1669810524 after retry RPC after error@815614139) -> (continuation of Deferred@125280409 after retry RPC after error@1922107469) -> (continuation of Deferred@730165994 after retry RPC after error@1904689073) -> (continuation of Deferred@1263417968 after retry RPC after error@837255220) -> (continuation of Deferred@838186688 after retry RPC after error@2034728440) -> (continuation of Deferred@1138771226 after retry RPC after error@1309383538) -> (continuation of Deferred@1389656005 after retry RPC after error@280914138) -> (continuation of Deferred@1370957634 after retry RPC after error@1140192781) -> (continuation of Deferred@846126057 after retry RPC after error@1375051206) -> (continuation of Deferred@965750157 after retry RPC after error@1438245164) -> (continuation of Deferred@290249185 after retry RPC after error@1823112611) -> (continuation of Deferred@1416546814 after retry RPC after error@1321530078) -> (continuation of Deferred@967878966 after retry RPC after error@113810630) -> (continuation of Deferred@617321688 after retry RPC after error@1307627677) -> (continuation of Deferred@1899309324 after retry RPC after error@869905728) -> (continuation of Deferred@1263307447 after retry RPC after error@7593851) -> (continuation of Deferred@1482095408 after retry RPC after error@651896527) -> (continuation of Deferred@158140817 after retry RPC after error@402364352)))
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=36, TimeoutTracker(timeout=50000, elapsed=48608), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [2ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [3ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [6ms] refreshing cache from master, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [7ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [8ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [8ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [8ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [26ms] refreshing cache from master, [26ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [27ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [28ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [28ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [29ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [29ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [46ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [48ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [48ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [49ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [66ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [67ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [68ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [68ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [68ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [69ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [86ms] refreshing cache from master, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [88ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [88ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [88ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [89ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [166ms] refreshing cache from master, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [167ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [168ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [168ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [168ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [206ms] refreshing cache from master, [206ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [207ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [208ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [208ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [209ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [209ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [326ms] refreshing cache from master, [327ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [327ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [327ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [328ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [328ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [329ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [329ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [846ms] refreshing cache from master, [846ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [846ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [848ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [848ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [848ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [848ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [1826ms] refreshing cache from master, [1826ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [1826ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [1827ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [1827ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [1828ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [1828ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [1828ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [3566ms] refreshing cache from master, [3567ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [3567ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [3567ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [3568ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.188:43579: Network error: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579, [3568ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.189:34439: Network error: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [3569ms] Sub RPC ConnectToMaster: received response from server master-127.8.104.190:42679: OK, [3569ms] delaying RPC due to: Service unavailable: Master config (127.8.104.190:42679,127.8.104.188:43579,127.8.104.189:34439) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.188:43579(127.8.104.188:43579): Connection refused: /127.8.104.188:43579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.104.189:34439(127.8.104.189:34439): Connection refused: /127.8.104.189:34439, [4466ms] refreshing cache from master, [4466ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.190:42679, [4466ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.188:43579, [4467ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.104.189:34439, [4467ms] too many traces: truncated at 100 traces, deferred=Deferred@1302957249(state=PENDING, result=null, callback=(continuation of Deferred@581007372 after retry RPC after error@1759754833) -> (continuation of Deferred@1607695714 after retry RPC after error@1578970589) -> (continuation of Deferred@2002079498 after retry RPC after error@43078275) -> (continuation of Deferred@1189939166 after retry RPC after error@1906948379) -> (continuation of Deferred@752452923 after retry RPC after error@609537897) -> (continuation of Deferred@178913226 after retry RPC after error@922021605) -> (continuation of Deferred@1457804637 after retry RPC after error@1422522223) -> (continuation of Deferred@291177875 after retry RPC after error@13193883) -> (continuation of Deferred@1962127714 after retry RPC after error@1872384540) -> (continuation of Deferred@648324577 after retry RPC after error@1815307812) -> (continuation of Deferred@86159969 after retry RPC after error@940903019) -> (continuation of Deferred@991767333 after retry RPC after error@1290303524) -> (continuation of Deferred@1548319330 after retry RPC after error@1285514706) -> (continuation of Deferred@512310119 after retry RPC after error@296704458) -> (continuation of Deferred@102939639 after retry RPC after error@832892692) -> (continuation of Deferred@1713481796 after retry RPC after error@229362755) -> (continuation of Deferred@516236034 after retry RPC after error@143635329) -> (continuation of Deferred@1669810524 after retry RPC after error@815614139) -> (continuation of Deferred@125280409 after retry RPC after error@1922107469) -> (continuation of Deferred@730165994 after retry RPC after error@1904689073) -> (continuation of Deferred@1263417968 after retry RPC after error@837255220) -> (continuation of Deferred@838186688 after retry RPC after error@2034728440) -> (continuation of Deferred@1138771226 after retry RPC after error@1309383538) -> (continuation of Deferred@1389656005 after retry RPC after error@280914138) -> (continuation of Deferred@1370957634 after retry RPC after error@1140192781) -> (continuation of Deferred@846126057 after retry RPC after error@1375051206) -> (continuation of Deferred@965750157 after retry RPC after error@1438245164) -> (continuation of Deferred@290249185 after retry RPC after error@1823112611) -> (continuation of Deferred@1416546814 after retry RPC after error@1321530078) -> (continuation of Deferred@967878966 after retry RPC after error@113810630) -> (continuation of Deferred@617321688 after retry RPC after error@1307627677) -> (continuation of Deferred@1899309324 after retry RPC after error@869905728) -> (continuation of Deferred@1263307447 after retry RPC after error@7593851) -> (continuation of Deferred@1482095408 after retry RPC after error@651896527) -> (continuation of Deferred@158140817 after retry RPC after error@402364352), errback=(continuation of Deferred@581007372 after retry RPC after error@1759754833) -> (continuation of Deferred@1607695714 after retry RPC after error@1578970589) -> (continuation of Deferred@2002079498 after retry RPC after error@43078275) -> (continuation of Deferred@1189939166 after retry RPC after error@1906948379) -> (continuation of Deferred@752452923 after retry RPC after error@609537897) -> (continuation of Deferred@178913226 after retry RPC after error@922021605) -> (continuation of Deferred@1457804637 after retry RPC after error@1422522223) -> (continuation of Deferred@291177875 after retry RPC after error@13193883) -> (continuation of Deferred@1962127714 after retry RPC after error@1872384540) -> (continuation of Deferred@648324577 after retry RPC after error@1815307812) -> (continuation of Deferred@86159969 after retry RPC after error@940903019) -> (continuation of Deferred@991767333 after retry RPC after error@1290303524) -> (continuation of Deferred@1548319330 after retry RPC after error@1285514706) -> (continuation of Deferred@512310119 after retry RPC after error@296704458) -> (continuation of Deferred@102939639 after retry RPC after error@832892692) -> (continuation of Deferred@1713481796 after retry RPC after error@229362755) -> (continuation of Deferred@516236034 after retry RPC after error@143635329) -> (continuation of Deferred@1669810524 after retry RPC after error@815614139) -> (continuation of Deferred@125280409 after retry RPC after error@1922107469) -> (continuation of Deferred@730165994 after retry RPC after error@1904689073) -> (continuation of Deferred@1263417968 after retry RPC after error@837255220) -> (continuation of Deferred@838186688 after retry RPC after error@2034728440) -> (continuation of Deferred@1138771226 after retry RPC after error@1309383538) -> (continuation of Deferred@1389656005 after retry RPC after error@280914138) -> (continuation of Deferred@1370957634 after retry RPC after error@1140192781) -> (continuation of Deferred@846126057 after retry RPC after error@1375051206) -> (continuation of Deferred@965750157 after retry RPC after error@1438245164) -> (continuation of Deferred@290249185 after retry RPC after error@1823112611) -> (continuation of Deferred@1416546814 after retry RPC after error@1321530078) -> (continuation of Deferred@967878966 after retry RPC after error@113810630) -> (continuation of Deferred@617321688 after retry RPC after error@1307627677) -> (continuation of Deferred@1899309324 after retry RPC after error@869905728) -> (continuation of Deferred@1263307447 after retry RPC after error@7593851) -> (continuation of Deferred@1482095408 after retry RPC after error@651896527) -> (continuation of Deferred@158140817 after retry RPC after error@402364352)))
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)