02:49:10.327 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:49:10.327 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQjj_5e/test-tmp
02:49:10.327 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
02:49:10.328 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQjj_5e/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
02:49:10.329 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:49:10.660 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:49:10.660 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:49:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:10.662814 25959 mini_oidc.cc:152] Starting JWKS server
02:49:10.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:10.665786 25959 webserver.cc:460] Webserver started at http://127.0.0.1:36465/ using document root <none> and password file <none>
02:49:10.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:10.666024 25959 mini_oidc.cc:189] Starting OIDC Discovery server
02:49:10.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:10.666625 25959 webserver.cc:460] Webserver started at http://127.0.0.1:45113/ using document root <none> and password file <none>
02:49:10.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T02:49:10Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:49:10.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T02:49:10Z Disabled control of system clock
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:10.697474 25959 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/wal
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/data
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/logs
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/data/info.pb
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:49:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.254:40383
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.254
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36465/jwks.json
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
02:49:10.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
02:49:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.027468 25970 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:49:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.027899 25970 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:49:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.028000 25970 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:49:11.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.036244 25970 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:49:11.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.036371 25970 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
02:49:11.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.036408 25970 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:49:11.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.036455 25970 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:49:11.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.036491 25970 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:49:11.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.036543 25970 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:49:11.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.036574 25970 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36465/jwks.json
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.046406 25970 master_runner.cc:386] Master server non-default flags:
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/data
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/wal
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.254:40383
02:49:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36465/jwks.json
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/data/info.pb
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.254
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/logs
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 01:54:22 UTC on ccycloud.quasar-vafwje.root.comops.site
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454977
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:49:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.047724 25970 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:49:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.049616 25970 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:49:11.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.061955 25975 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:11.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.061980 25976 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:11.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.061946 25978 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:11.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.063253 25970 server_base.cc:1034] running on GCE node
02:49:11.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.064083 25970 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:49:11.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.065788 25970 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:49:11.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.067065 25970 hybrid_clock.cc:648] HybridClock initialized: now 1781837351066996 us; error 41 us; skew 500 ppm
02:49:11.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.067489 25970 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:49:11.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.070016 25970 webserver.cc:460] Webserver started at http://127.25.89.254:37715/ using document root <none> and password file <none>
02:49:11.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.070770 25970 fs_manager.cc:359] Metadata directory not provided
02:49:11.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.070878 25970 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:49:11.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.071209 25970 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:49:11.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.074066 25970 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/data/instance:
02:49:11.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3af63bec99064cbbb230e2f985fd1a34"
02:49:11.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:11 on dist-test-slave-6wc9"
02:49:11.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.074915 25970 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/wal/instance:
02:49:11.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3af63bec99064cbbb230e2f985fd1a34"
02:49:11.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:11 on dist-test-slave-6wc9"
02:49:11.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.080580 25970 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.001s
02:49:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.085529 25984 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:49:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.087409 25970 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:49:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.087617 25970 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/data,/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/wal
02:49:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3af63bec99064cbbb230e2f985fd1a34"
02:49:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:11 on dist-test-slave-6wc9"
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.087882 25970 fs_report.cc:383] FS layout report
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/wal
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/wal
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/data/data
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:49:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:49:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.103394 25970 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:49:11.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.104368 25970 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:49:11.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.104705 25970 kserver.cc:163] Server-wide thread pool size limit: 3276
02:49:11.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.125535 25970 rpc_server.cc:307] RPC server started. Bound to: 127.25.89.254:40383
02:49:11.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.125555 26045 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.89.254:40383 every 8 connection(s)
02:49:11.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.127022 25970 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/data/info.pb
02:49:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.131038 26046 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.135464 25959 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu as pid 25970
02:49:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.135763 25959 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-0/wal/instance
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.138767 25959 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/wal
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/data
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/logs
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/data/info.pb
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.253:46647
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.253
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36465/jwks.json
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
02:49:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
02:49:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.138962 26046 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } has no permanent_uuid. Determining permanent_uuid...
02:49:11.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.158201 26046 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } has no permanent_uuid. Determining permanent_uuid...
02:49:11.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.159585 25985 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.161243 26046 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:11.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.210609 26046 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } attempt: 1
02:49:11.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.212778 26046 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.281111 26046 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } attempt: 2
02:49:11.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.283382 26046 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:11.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.374652 26046 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } attempt: 3
02:49:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.376571 26046 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.475237 26048 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:49:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.475538 26048 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:49:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.475621 26048 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:49:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.482990 26048 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:49:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.483107 26048 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
02:49:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.483142 26048 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:49:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.483187 26048 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:49:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.483223 26048 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:49:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.483273 26048 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:49:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.483304 26048 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36465/jwks.json
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.492939 26048 master_runner.cc:386] Master server non-default flags:
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/data
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/wal
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.253:46647
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36465/jwks.json
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/data/info.pb
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.253
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/logs
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:49:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:49:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:49:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:49:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:49:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:49:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 01:54:22 UTC on ccycloud.quasar-vafwje.root.comops.site
02:49:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454977
02:49:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:49:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.494295 26048 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:49:11.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.496160 26048 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:49:11.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.509671 26056 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:11.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.509758 26059 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:11.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.509702 26057 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:11.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.510923 26048 server_base.cc:1034] running on GCE node
02:49:11.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.511773 26048 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:49:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.513391 26048 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:49:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.514613 26048 hybrid_clock.cc:648] HybridClock initialized: now 1781837351514569 us; error 36 us; skew 500 ppm
02:49:11.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.514976 26048 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:49:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.517567 26048 webserver.cc:460] Webserver started at http://127.25.89.253:34847/ using document root <none> and password file <none>
02:49:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.518246 26048 fs_manager.cc:359] Metadata directory not provided
02:49:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.518335 26048 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:49:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.518599 26048 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:49:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.519819 26046 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } attempt: 4
02:49:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.521880 26046 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.521862 26048 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/data/instance:
02:49:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c0f8ffb53444dc5bddc487da2466d59"
02:49:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:11 on dist-test-slave-6wc9"
02:49:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.522684 26048 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/wal/instance:
02:49:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c0f8ffb53444dc5bddc487da2466d59"
02:49:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:11 on dist-test-slave-6wc9"
02:49:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.528143 26048 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
02:49:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.532171 26065 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.533977 26048 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.534181 26048 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/data,/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/wal
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c0f8ffb53444dc5bddc487da2466d59"
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:11 on dist-test-slave-6wc9"
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.534405 26048 fs_report.cc:383] FS layout report
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/wal
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/wal
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/data/data
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:49:11.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:49:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:49:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:49:11.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.551508 26048 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:49:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.552397 26048 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:49:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.552654 26048 kserver.cc:163] Server-wide thread pool size limit: 3276
02:49:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.573387 26048 rpc_server.cc:307] RPC server started. Bound to: 127.25.89.253:46647
02:49:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.573393 26126 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.89.253:46647 every 8 connection(s)
02:49:11.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.574703 26048 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/data/info.pb
02:49:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.577545 25959 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu as pid 26048
02:49:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.577695 25959 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-1/wal/instance
02:49:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.578603 25959 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/wal
02:49:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/data
02:49:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:49:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:49:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/logs
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/data/info.pb
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.252:40643
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.252
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36465/jwks.json
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
02:49:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.578771 26127 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.585930 26127 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } has no permanent_uuid. Determining permanent_uuid...
02:49:11.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.603453 26127 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } has no permanent_uuid. Determining permanent_uuid...
02:49:11.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.614956 26127 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } has no permanent_uuid. Determining permanent_uuid...
02:49:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.616348 26067 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.618099 26127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.252:40643: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:11.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.667492 26127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } attempt: 1
02:49:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.669598 26127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.252:40643: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:11.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.737893 26127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } attempt: 2
02:49:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.740084 26127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.252:40643: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:11.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.821177 26046 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } attempt: 5
02:49:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.828013 26046 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } has no permanent_uuid. Determining permanent_uuid...
02:49:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.829932 26046 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.252:40643: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:11.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.831321 26127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } attempt: 3
02:49:11.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.833017 26127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.252:40643: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.881274 26046 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } attempt: 1
02:49:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.883392 26046 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.252:40643: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:11.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.919452 26129 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:49:11.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.919781 26129 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:49:11.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.919866 26129 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:49:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.927471 26129 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:49:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.927606 26129 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
02:49:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.927644 26129 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:49:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.927688 26129 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:49:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.927724 26129 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:49:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.927788 26129 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:49:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.927821 26129 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36465/jwks.json
02:49:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.937489 26129 master_runner.cc:386] Master server non-default flags:
02:49:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
02:49:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/data
02:49:11.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/wal
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.252:40643
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36465/jwks.json
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/data/info.pb
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.252
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/logs
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 01:54:22 UTC on ccycloud.quasar-vafwje.root.comops.site
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454977
02:49:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:49:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.938902 26129 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:49:11.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.940846 26129 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:49:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.951676 26046 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } attempt: 2
02:49:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.953419 26138 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:11.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.954099 26046 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.252:40643: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:11.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.954202 26137 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:11.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.953462 26140 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.955065 26129 server_base.cc:1034] running on GCE node
02:49:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.955858 26129 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:49:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.957463 26129 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:49:11.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.958685 26129 hybrid_clock.cc:648] HybridClock initialized: now 1781837351958608 us; error 70 us; skew 500 ppm
02:49:11.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.959077 26129 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:49:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.961596 26129 webserver.cc:460] Webserver started at http://127.25.89.252:43609/ using document root <none> and password file <none>
02:49:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.962371 26129 fs_manager.cc:359] Metadata directory not provided
02:49:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.962478 26129 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:49:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.962826 26129 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:49:11.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.965695 26129 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/data/instance:
02:49:11.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57d79a0a4f074da4b31478e6fdaa1954"
02:49:11.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:11 on dist-test-slave-6wc9"
02:49:11.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.966548 26129 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/wal/instance:
02:49:11.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57d79a0a4f074da4b31478e6fdaa1954"
02:49:11.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:11 on dist-test-slave-6wc9"
02:49:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.972078 26129 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
02:49:11.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.975837 26146 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:49:11.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.976229 26127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } attempt: 4
02:49:11.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.977560 26129 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.977757 26129 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/data,/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/wal
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57d79a0a4f074da4b31478e6fdaa1954"
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:11 on dist-test-slave-6wc9"
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:11.978078 26127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.89.252:40643: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.977998 26129 fs_report.cc:383] FS layout report
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/wal
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/wal
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/data/data
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:49:11.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:49:11.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.998497 26129 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:49:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.999497 26129 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:49:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:11.999862 26129 kserver.cc:163] Server-wide thread pool size limit: 3276
02:49:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.021519 26207 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.89.252:40643 every 8 connection(s)
02:49:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.021538 26129 rpc_server.cc:307] RPC server started. Bound to: 127.25.89.252:40643
02:49:12.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.023087 26129 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/data/info.pb
02:49:12.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.026129 25959 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu as pid 26129
02:49:12.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.026311 25959 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/master-2/wal/instance
02:49:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.028033 26208 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:12.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.037561 26208 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } has no permanent_uuid. Determining permanent_uuid...
02:49:12.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.053846 26208 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } has no permanent_uuid. Determining permanent_uuid...
02:49:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.060319 26046 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } attempt: 3
02:49:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.062319 26208 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } has no permanent_uuid. Determining permanent_uuid...
02:49:12.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.072100 26046 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34: Bootstrap starting.
02:49:12.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.073411 26208 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954: Bootstrap starting.
02:49:12.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.075531 26046 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34: Neither blocks nor log segments found. Creating new log.
02:49:12.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.076697 26046 log.cc:824] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34: Log is configured to *not* fsync() on all Append() calls
02:49:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.076803 26208 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954: Neither blocks nor log segments found. Creating new log.
02:49:12.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.078173 26208 log.cc:824] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954: Log is configured to *not* fsync() on all Append() calls
02:49:12.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.080420 26046 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34: No bootstrap required, opened a new log
02:49:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.083577 26208 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954: No bootstrap required, opened a new log
02:49:12.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.088490 26046 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:12.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.088807 26046 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:12.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.088882 26046 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3af63bec99064cbbb230e2f985fd1a34, State: Initialized, Role: FOLLOWER
02:49:12.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.089520 26046 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.090390 26217 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.090816 26217 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [sys.catalog]: This master's current role is: FOLLOWER
02:49:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.092072 26046 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [sys.catalog]: configured and running, proceeding with master startup.
02:49:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.092304 26208 raft_consensus.cc:348] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.092687 26208 raft_consensus.cc:374] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.092798 26208 raft_consensus.cc:729] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57d79a0a4f074da4b31478e6fdaa1954, State: Initialized, Role: FOLLOWER
02:49:12.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.093430 26208 consensus_queue.cc:260] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:12.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.094295 26221 sys_catalog.cc:455] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:12.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.094691 26221 sys_catalog.cc:458] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [sys.catalog]: This master's current role is: FOLLOWER
02:49:12.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.095587 26208 sys_catalog.cc:564] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [sys.catalog]: configured and running, proceeding with master startup.
02:49:12.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.103741 26230 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:49:12.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.104169 26230 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:49:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.107831 25990 tablet.cc:2369] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:49:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.101297 25970 master_runner.cc:186] Error getting master registration for 127.25.89.253:46647: OK, instance_id { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" instance_seqno: 1781837351568381 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:49:12.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.112877 25970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:49:12.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.118669 26241 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:49:12.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.118819 26241 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:49:12.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.117656 26129 master_runner.cc:186] Error getting master registration for 127.25.89.253:46647: OK, instance_id { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" instance_seqno: 1781837351568381 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:49:12.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.126371 26129 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:49:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.137547 26221 raft_consensus.cc:482] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:49:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.137909 26221 raft_consensus.cc:504] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.139699 26221 leader_election.cc:288] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3af63bec99064cbbb230e2f985fd1a34 (127.25.89.254:40383), 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647)
02:49:12.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.140887 26011 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "57d79a0a4f074da4b31478e6fdaa1954" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3af63bec99064cbbb230e2f985fd1a34" is_pre_election: true
02:49:12.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.141602 26011 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 57d79a0a4f074da4b31478e6fdaa1954 in term 0.
02:49:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.142731 26150 leader_election.cc:302] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34, 57d79a0a4f074da4b31478e6fdaa1954; no voters:
02:49:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.143290 26221 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:49:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.143453 26221 raft_consensus.cc:482] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:49:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.143565 26221 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [term 0 FOLLOWER]: Advancing to term 1
02:49:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.146884 26221 raft_consensus.cc:504] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.148444 26011 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "57d79a0a4f074da4b31478e6fdaa1954" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3af63bec99064cbbb230e2f985fd1a34"
02:49:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.148705 26011 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 0 FOLLOWER]: Advancing to term 1
02:49:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.149087 26221 leader_election.cc:288] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [CANDIDATE]: Term 1 election: Requested vote from peers 3af63bec99064cbbb230e2f985fd1a34 (127.25.89.254:40383), 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647)
02:49:12.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.152114 26011 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57d79a0a4f074da4b31478e6fdaa1954 in term 1.
02:49:12.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.152765 26150 leader_election.cc:302] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34, 57d79a0a4f074da4b31478e6fdaa1954; no voters:
02:49:12.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.153223 26221 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [term 1 FOLLOWER]: Leader election won for term 1
02:49:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.153978 26221 raft_consensus.cc:686] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [term 1 LEADER]: Becoming Leader. State: Replica: 57d79a0a4f074da4b31478e6fdaa1954, State: Running, Role: LEADER
02:49:12.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.154672 26221 consensus_queue.cc:237] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:12.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.168743 26243 sys_catalog.cc:455] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 57d79a0a4f074da4b31478e6fdaa1954. Latest consensus state: current_term: 1 leader_uuid: "57d79a0a4f074da4b31478e6fdaa1954" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:12.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.170122 26243 sys_catalog.cc:458] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [sys.catalog]: This master's current role is: LEADER
02:49:12.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.171768 26245 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
02:49:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.174242 26245 catalog_manager.cc:1484] Initializing Kudu cluster ID...
02:49:12.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.188956 26011 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 1 FOLLOWER]: Refusing update from remote peer 57d79a0a4f074da4b31478e6fdaa1954: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:49:12.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.190219 26221 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:49:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.211217 26217 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 57d79a0a4f074da4b31478e6fdaa1954. Latest consensus state: current_term: 1 leader_uuid: "57d79a0a4f074da4b31478e6fdaa1954" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.211581 26217 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [sys.catalog]: This master's current role is: FOLLOWER
02:49:12.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.220358 26221 sys_catalog.cc:455] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "57d79a0a4f074da4b31478e6fdaa1954" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:12.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.220715 26221 sys_catalog.cc:458] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [sys.catalog]: This master's current role is: LEADER
02:49:12.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.222281 26245 catalog_manager.cc:1347] Generated new cluster ID: 560d1240bf214b3398c335ec5b2de248
02:49:12.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.222396 26245 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
02:49:12.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.230463 26247 mvcc.cc:205] Tried to move back new op lower bound from 7298405794549784576 to 7298405794431676416. Current Snapshot: MvccSnapshot[applied={T|T < 7298405794549784576}]
02:49:12.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.234134 26217 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "57d79a0a4f074da4b31478e6fdaa1954" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:12.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.234462 26217 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [sys.catalog]: This master's current role is: FOLLOWER
02:49:12.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.272888 26245 catalog_manager.cc:1370] Generated new certificate authority record
02:49:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.275063 26245 catalog_manager.cc:1504] Loading token signing keys...
02:49:12.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.277253 26127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } attempt: 5
02:49:12.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.301929 26245 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954: Generated new TSK 0
02:49:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.303159 26245 catalog_manager.cc:1514] Initializing in-progress tserver states...
02:49:12.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.305389 26127 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59: Bootstrap starting.
02:49:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.314960 26127 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59: Neither blocks nor log segments found. Creating new log.
02:49:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.318205 26127 log.cc:824] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59: Log is configured to *not* fsync() on all Append() calls
02:49:12.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.326365 26127 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59: No bootstrap required, opened a new log
02:49:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.329367 25959 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/wal
02:49:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/data
02:49:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/logs
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/data/info.pb
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.193:0
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.89.193
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.193
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:49:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
02:49:12.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.351241 26127 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:12.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.352272 26127 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:12.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.352587 26127 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c0f8ffb53444dc5bddc487da2466d59, State: Initialized, Role: FOLLOWER
02:49:12.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.354195 26127 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:12.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.365576 26253 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:12.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.366070 26253 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [sys.catalog]: This master's current role is: FOLLOWER
02:49:12.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.369128 26091 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "57d79a0a4f074da4b31478e6fdaa1954" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c0f8ffb53444dc5bddc487da2466d59"
02:49:12.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.369678 26091 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 0 FOLLOWER]: Advancing to term 1
02:49:12.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.372957 26091 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 57d79a0a4f074da4b31478e6fdaa1954 in term 1.
02:49:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.374214 26092 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "57d79a0a4f074da4b31478e6fdaa1954" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c0f8ffb53444dc5bddc487da2466d59" is_pre_election: true
02:49:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.374971 26090 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 1 FOLLOWER]: Refusing update from remote peer 57d79a0a4f074da4b31478e6fdaa1954: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:49:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.376302 26127 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [sys.catalog]: configured and running, proceeding with master startup.
02:49:12.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.376979 26243 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:49:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.439424 26253 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 57d79a0a4f074da4b31478e6fdaa1954. Latest consensus state: current_term: 1 leader_uuid: "57d79a0a4f074da4b31478e6fdaa1954" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.439857 26253 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [sys.catalog]: This master's current role is: FOLLOWER
02:49:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.439453 26258 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "57d79a0a4f074da4b31478e6fdaa1954" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.439965 26258 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [sys.catalog]: This master's current role is: FOLLOWER
02:49:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.461783 26243 sys_catalog.cc:455] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "57d79a0a4f074da4b31478e6fdaa1954" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.462154 26243 sys_catalog.cc:458] T 00000000000000000000000000000000 P 57d79a0a4f074da4b31478e6fdaa1954 [sys.catalog]: This master's current role is: LEADER
02:49:12.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.465869 26269 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:49:12.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.790263 26252 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:49:12.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.790567 26252 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:49:12.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.790670 26252 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:49:12.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.798946 26252 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:49:12.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.799100 26252 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
02:49:12.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.799137 26252 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
02:49:12.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.799181 26252 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
02:49:12.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.799245 26252 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.89.193
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.809329 26252 tablet_server_runner.cc:78] Tablet server non-default flags:
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/data
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/wal
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.193:0
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/data/info.pb
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.193
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.89.193
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/logs
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 01:54:22 UTC on ccycloud.quasar-vafwje.root.comops.site
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454977
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:49:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.810679 26252 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:49:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.812465 26252 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:49:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.826777 26275 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.826808 26274 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:12.827450 26277 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.827703 26252 server_base.cc:1034] running on GCE node
02:49:12.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.828363 26252 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:49:12.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.829456 26252 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:49:12.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.830698 26252 hybrid_clock.cc:648] HybridClock initialized: now 1781837352830640 us; error 47 us; skew 500 ppm
02:49:12.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.831060 26252 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:49:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.833604 26252 webserver.cc:460] Webserver started at http://127.25.89.193:44967/ using document root <none> and password file <none>
02:49:12.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.834354 26252 fs_manager.cc:359] Metadata directory not provided
02:49:12.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.834468 26252 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:49:12.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.834805 26252 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:49:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.837697 26252 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/data/instance:
02:49:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60171f79b6734d19a4b20f9b7f5ce54a"
02:49:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:12 on dist-test-slave-6wc9"
02:49:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.838639 26252 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/wal/instance:
02:49:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60171f79b6734d19a4b20f9b7f5ce54a"
02:49:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:12 on dist-test-slave-6wc9"
02:49:12.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.844358 26252 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
02:49:12.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.848168 26283 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:49:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.849838 26252 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.850008 26252 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/data,/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/wal
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60171f79b6734d19a4b20f9b7f5ce54a"
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:12 on dist-test-slave-6wc9"
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.850217 26252 fs_report.cc:383] FS layout report
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/wal
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/wal
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/data/data
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:49:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:49:12.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.868275 26252 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:49:12.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.869155 26252 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:49:12.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.869500 26252 kserver.cc:163] Server-wide thread pool size limit: 3276
02:49:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.872400 26252 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:49:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.872494 26252 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:49:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.872627 26252 ts_tablet_manager.cc:610] Registered 0 tablets
02:49:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.872687 26252 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:49:12.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.927399 26252 rpc_server.cc:307] RPC server started. Bound to: 127.25.89.193:35885
02:49:12.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.927434 26407 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.89.193:35885 every 8 connection(s)
02:49:12.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.928697 26252 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/data/info.pb
02:49:12.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.931627 26409 heartbeater.cc:347] Connected to a master server at 127.25.89.254:40383
02:49:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.931953 26409 heartbeater.cc:464] Registering TS with master...
02:49:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.932817 26409 heartbeater.cc:511] Master 127.25.89.254:40383 requested a full tablet report, sending...
02:49:12.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.934643 25959 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu as pid 26252
02:49:12.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.934837 25959 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-0/wal/instance
02:49:12.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.935330 26001 ts_manager.cc:194] Registered new tserver with Master: 60171f79b6734d19a4b20f9b7f5ce54a (127.25.89.193:35885)
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.939674 25959 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/wal
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/data
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/logs
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/data/info.pb
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.194:0
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.89.194
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.194
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:49:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:49:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
02:49:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.941427 26408 heartbeater.cc:347] Connected to a master server at 127.25.89.252:40643
02:49:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.941570 26408 heartbeater.cc:464] Registering TS with master...
02:49:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.941895 26408 heartbeater.cc:511] Master 127.25.89.252:40643 requested a full tablet report, sending...
02:49:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.942564 26410 heartbeater.cc:347] Connected to a master server at 127.25.89.253:46647
02:49:12.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.942708 26410 heartbeater.cc:464] Registering TS with master...
02:49:12.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.943094 26410 heartbeater.cc:511] Master 127.25.89.253:46647 requested a full tablet report, sending...
02:49:12.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.943509 26163 ts_manager.cc:194] Registered new tserver with Master: 60171f79b6734d19a4b20f9b7f5ce54a (127.25.89.193:35885)
02:49:12.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.944752 26082 ts_manager.cc:194] Registered new tserver with Master: 60171f79b6734d19a4b20f9b7f5ce54a (127.25.89.193:35885)
02:49:12.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:12.945513 26163 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.25.89.193:49221
02:49:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.106345 26230 catalog_manager.cc:1259] Loaded cluster ID: 560d1240bf214b3398c335ec5b2de248
02:49:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.106447 26230 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34: loading cluster ID for follower catalog manager: success
02:49:13.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.109499 26230 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34: acquiring CA information for follower catalog manager: success
02:49:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.111403 26230 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:49:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.294281 26413 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:49:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.294616 26413 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:49:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.294725 26413 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:49:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.302781 26413 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:49:13.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.302928 26413 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
02:49:13.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.302963 26413 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
02:49:13.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.303005 26413 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
02:49:13.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.303076 26413 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.89.194
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.312772 26413 tablet_server_runner.cc:78] Tablet server non-default flags:
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/data
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/wal
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.194:0
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/data/info.pb
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.194
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.89.194
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/logs
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 01:54:22 UTC on ccycloud.quasar-vafwje.root.comops.site
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454977
02:49:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:49:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.314188 26413 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:49:13.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.316085 26413 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:49:13.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.330489 26418 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.330401 26421 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.330427 26419 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.334955 26413 server_base.cc:1034] running on GCE node
02:49:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.335714 26413 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:49:13.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.336830 26413 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:49:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.338050 26413 hybrid_clock.cc:648] HybridClock initialized: now 1781837353337988 us; error 57 us; skew 500 ppm
02:49:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.338439 26413 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:49:13.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.340958 26413 webserver.cc:460] Webserver started at http://127.25.89.194:41829/ using document root <none> and password file <none>
02:49:13.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.341750 26413 fs_manager.cc:359] Metadata directory not provided
02:49:13.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.341848 26413 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:49:13.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.342123 26413 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:49:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.344915 26413 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/data/instance:
02:49:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95899bff95504c12969daf319b305f5c"
02:49:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:13 on dist-test-slave-6wc9"
02:49:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.345799 26413 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/wal/instance:
02:49:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95899bff95504c12969daf319b305f5c"
02:49:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:13 on dist-test-slave-6wc9"
02:49:13.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.351619 26413 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
02:49:13.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.355494 26427 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.357018 26413 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.001s
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.357239 26413 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/data,/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/wal
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95899bff95504c12969daf319b305f5c"
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:13 on dist-test-slave-6wc9"
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.357452 26413 fs_report.cc:383] FS layout report
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/wal
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/wal
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/data/data
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:49:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:49:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.383286 26413 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:49:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.384194 26413 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:49:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.384473 26413 kserver.cc:163] Server-wide thread pool size limit: 3276
02:49:13.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.387495 26413 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:49:13.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.387610 26413 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:49:13.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.387753 26413 ts_tablet_manager.cc:610] Registered 0 tablets
02:49:13.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.387813 26413 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:49:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.448321 26413 rpc_server.cc:307] RPC server started. Bound to: 127.25.89.194:41711
02:49:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.448347 26548 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.89.194:41711 every 8 connection(s)
02:49:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.449630 26413 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/data/info.pb
02:49:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.450459 25959 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu as pid 26413
02:49:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.450616 25959 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-1/wal/instance
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.457612 25959 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/wal
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/data
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/logs
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/data/info.pb
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.195:0
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.89.195
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.195
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:49:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:49:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
02:49:13.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.465267 26550 heartbeater.cc:347] Connected to a master server at 127.25.89.254:40383
02:49:13.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.465927 26550 heartbeater.cc:464] Registering TS with master...
02:49:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.466931 26550 heartbeater.cc:511] Master 127.25.89.254:40383 requested a full tablet report, sending...
02:49:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.465366 26549 heartbeater.cc:347] Connected to a master server at 127.25.89.252:40643
02:49:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.467813 26549 heartbeater.cc:464] Registering TS with master...
02:49:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.468225 26549 heartbeater.cc:511] Master 127.25.89.252:40643 requested a full tablet report, sending...
02:49:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.469376 26163 ts_manager.cc:194] Registered new tserver with Master: 95899bff95504c12969daf319b305f5c (127.25.89.194:41711)
02:49:13.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.469798 26551 heartbeater.cc:347] Connected to a master server at 127.25.89.253:46647
02:49:13.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.469833 26001 ts_manager.cc:194] Registered new tserver with Master: 95899bff95504c12969daf319b305f5c (127.25.89.194:41711)
02:49:13.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.469919 26551 heartbeater.cc:464] Registering TS with master...
02:49:13.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.470286 26551 heartbeater.cc:511] Master 127.25.89.253:46647 requested a full tablet report, sending...
02:49:13.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.470458 26163 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.25.89.194:50163
02:49:13.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.471261 26082 ts_manager.cc:194] Registered new tserver with Master: 95899bff95504c12969daf319b305f5c (127.25.89.194:41711)
02:49:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.476778 26269 catalog_manager.cc:1259] Loaded cluster ID: 560d1240bf214b3398c335ec5b2de248
02:49:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.476929 26269 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59: loading cluster ID for follower catalog manager: success
02:49:13.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.481334 26269 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59: acquiring CA information for follower catalog manager: success
02:49:13.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.484400 26269 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:49:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.817368 26554 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:49:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.817690 26554 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:49:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.817791 26554 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:49:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.825656 26554 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:49:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.825848 26554 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
02:49:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.825908 26554 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
02:49:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.825970 26554 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
02:49:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.826067 26554 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.89.195
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.835737 26554 tablet_server_runner.cc:78] Tablet server non-default flags:
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.89.212:41129
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/data
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/wal
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.89.195:0
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/data/info.pb
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.89.195
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.89.254:40383,127.25.89.253:46647,127.25.89.252:40643
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.89.195
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/logs
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 01:54:22 UTC on ccycloud.quasar-vafwje.root.comops.site
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454977
02:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
02:49:13.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.837066 26554 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:49:13.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.839067 26554 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:49:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.853011 26561 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.852977 26560 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:13.852936 26563 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:49:13.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.854297 26554 server_base.cc:1034] running on GCE node
02:49:13.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.855003 26554 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:49:13.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.856129 26554 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:49:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.857304 26554 hybrid_clock.cc:648] HybridClock initialized: now 1781837353857251 us; error 57 us; skew 500 ppm
02:49:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.857681 26554 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:49:13.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.860195 26554 webserver.cc:460] Webserver started at http://127.25.89.195:34767/ using document root <none> and password file <none>
02:49:13.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.861002 26554 fs_manager.cc:359] Metadata directory not provided
02:49:13.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.861122 26554 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:49:13.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.861515 26554 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:49:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.864528 26554 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/data/instance:
02:49:13.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f16375da86945e093b5a7cba9f06fea"
02:49:13.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:13 on dist-test-slave-6wc9"
02:49:13.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.865406 26554 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/wal/instance:
02:49:13.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f16375da86945e093b5a7cba9f06fea"
02:49:13.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:13 on dist-test-slave-6wc9"
02:49:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.871014 26554 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
02:49:13.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.874862 26569 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:49:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.876444 26554 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
02:49:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.876627 26554 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/data,/tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/wal
02:49:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f16375da86945e093b5a7cba9f06fea"
02:49:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 02:49:13 on dist-test-slave-6wc9"
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.876829 26554 fs_report.cc:383] FS layout report
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/wal
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/wal
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/data/data
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:49:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:49:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.902524 26554 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:49:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.903385 26554 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:49:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.903666 26554 kserver.cc:163] Server-wide thread pool size limit: 3276
02:49:13.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.906810 26554 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:49:13.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.906903 26554 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:49:13.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.907016 26554 ts_tablet_manager.cc:610] Registered 0 tablets
02:49:13.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.907074 26554 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:49:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.948302 26408 heartbeater.cc:503] Master 127.25.89.252:40643 was elected leader, sending a full tablet report...
02:49:13.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.967758 26554 rpc_server.cc:307] RPC server started. Bound to: 127.25.89.195:44665
02:49:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.967836 26690 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.89.195:44665 every 8 connection(s)
02:49:13.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.969172 26554 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/data/info.pb
02:49:13.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.971396 26692 heartbeater.cc:347] Connected to a master server at 127.25.89.254:40383
02:49:13.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.971829 26692 heartbeater.cc:464] Registering TS with master...
02:49:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.972309 25959 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu as pid 26554
02:49:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.972488 25959 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQjj_5e/test-tmp/mini-kudu-cluster2892788138758314708/ts-2/wal/instance
02:49:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.973011 26692 heartbeater.cc:511] Master 127.25.89.254:40383 requested a full tablet report, sending...
02:49:13.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.975704 26001 ts_manager.cc:194] Registered new tserver with Master: 0f16375da86945e093b5a7cba9f06fea (127.25.89.195:44665)
02:49:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.981683 26691 heartbeater.cc:347] Connected to a master server at 127.25.89.252:40643
02:49:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.982154 26691 heartbeater.cc:464] Registering TS with master...
02:49:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.982736 26691 heartbeater.cc:511] Master 127.25.89.252:40643 requested a full tablet report, sending...
02:49:13.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.984225 26163 ts_manager.cc:194] Registered new tserver with Master: 0f16375da86945e093b5a7cba9f06fea (127.25.89.195:44665)
02:49:13.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.985358 26163 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.25.89.195:58619
02:49:13.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.988266 26693 heartbeater.cc:347] Connected to a master server at 127.25.89.253:46647
02:49:13.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.988401 26693 heartbeater.cc:464] Registering TS with master...
02:49:13.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.988729 26693 heartbeater.cc:511] Master 127.25.89.253:46647 requested a full tablet report, sending...
02:49:13.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:13.989816 26082 ts_manager.cc:194] Registered new tserver with Master: 0f16375da86945e093b5a7cba9f06fea (127.25.89.195:44665)
02:49:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.000011 25959 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
02:49:14.001 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:49:14.002 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:49:14.003 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.25.89.254"
port: 40383
}
bound_http_address {
host: "127.25.89.254"
port: 37715
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.25.89.253"
port: 46647
}
bound_http_address {
host: "127.25.89.253"
port: 34847
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.25.89.252"
port: 40643
}
bound_http_address {
host: "127.25.89.252"
port: 43609
}
}
}
02:49:14.003 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:49:14.004 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.25.89.193"
port: 35885
}
bound_http_address {
host: "127.25.89.193"
port: 44967
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.25.89.194"
port: 41711
}
bound_http_address {
host: "127.25.89.194"
port: 41829
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.25.89.195"
port: 44665
}
bound_http_address {
host: "127.25.89.195"
port: 34767
}
}
}
02:49:14.004 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:49:14.006 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:49:14.007 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:14.008 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:14.009 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.25.89.253:46647(127.25.89.253:46647)] Successfully connected to peer
02:49:14.010 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.89.254:40383(127.25.89.254:40383)] connecting to peer
02:49:14.011 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.25.89.253
02:49:14.011 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.25.89.252:40643(127.25.89.252:40643)] Successfully connected to peer
02:49:14.013 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.25.89.252
02:49:14.014 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.25.89.254:40383(127.25.89.254:40383)] Successfully connected to peer
02:49:14.016 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.25.89.254
02:49:14.036 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x12099e51, L:/127.0.0.1:35460 - R:/127.25.89.252:40643] using SASL/PLAIN
02:49:14.041 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:49:14.041 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=34), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [32ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: OK, deferred=null) after lookup
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.043694 26163 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35460:
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:14.044 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xe4e25e43, L:/127.0.0.1:49584 - R:/127.25.89.254:40383] using SASL/PLAIN
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
02:49:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
02:49:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
02:49:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
02:49:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:14.046381 26163 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:49:14.046 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x286d0ecf, L:/127.0.0.1:56298 - R:/127.25.89.253:46647] using SASL/PLAIN
02:49:14.060 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 6430e55dd2ab4e378ff92a4c488c280f
02:49:14.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.099741 26618 tablet_service.cc:1467] Processing CreateTablet for tablet 0d88dd7fe3274724aee1670794deb4ea (DEFAULT_TABLE table=txn_manager_fallback [id=6430e55dd2ab4e378ff92a4c488c280f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
02:49:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.101117 26619 tablet_service.cc:1467] Processing CreateTablet for tablet ddde257f75d543e99a5f14f9ffd7913c (DEFAULT_TABLE table=txn_manager_fallback [id=6430e55dd2ab4e378ff92a4c488c280f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
02:49:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.102355 26619 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddde257f75d543e99a5f14f9ffd7913c. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.106124 26618 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0d88dd7fe3274724aee1670794deb4ea. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.107146 26477 tablet_service.cc:1467] Processing CreateTablet for tablet 0d88dd7fe3274724aee1670794deb4ea (DEFAULT_TABLE table=txn_manager_fallback [id=6430e55dd2ab4e378ff92a4c488c280f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
02:49:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.108806 26477 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0d88dd7fe3274724aee1670794deb4ea. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.109522 26478 tablet_service.cc:1467] Processing CreateTablet for tablet ddde257f75d543e99a5f14f9ffd7913c (DEFAULT_TABLE table=txn_manager_fallback [id=6430e55dd2ab4e378ff92a4c488c280f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
02:49:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.110042 26478 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddde257f75d543e99a5f14f9ffd7913c. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:14.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.118319 26334 tablet_service.cc:1467] Processing CreateTablet for tablet ddde257f75d543e99a5f14f9ffd7913c (DEFAULT_TABLE table=txn_manager_fallback [id=6430e55dd2ab4e378ff92a4c488c280f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
02:49:14.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.120223 26334 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddde257f75d543e99a5f14f9ffd7913c. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:14.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.118294 26333 tablet_service.cc:1467] Processing CreateTablet for tablet 0d88dd7fe3274724aee1670794deb4ea (DEFAULT_TABLE table=txn_manager_fallback [id=6430e55dd2ab4e378ff92a4c488c280f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
02:49:14.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.121845 26333 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0d88dd7fe3274724aee1670794deb4ea. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.127588 26717 tablet_bootstrap.cc:492] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c: Bootstrap starting.
02:49:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.133301 26717 tablet_bootstrap.cc:654] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c: Neither blocks nor log segments found. Creating new log.
02:49:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.134907 26717 log.cc:824] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c: Log is configured to *not* fsync() on all Append() calls
02:49:14.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.136363 26716 tablet_bootstrap.cc:492] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea: Bootstrap starting.
02:49:14.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.141439 26716 tablet_bootstrap.cc:654] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea: Neither blocks nor log segments found. Creating new log.
02:49:14.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.143702 26716 log.cc:824] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea: Log is configured to *not* fsync() on all Append() calls
02:49:14.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.145574 26719 tablet_bootstrap.cc:492] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a: Bootstrap starting.
02:49:14.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.147033 26717 tablet_bootstrap.cc:492] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c: No bootstrap required, opened a new log
02:49:14.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.147363 26717 ts_tablet_manager.cc:1397] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c: Time spent bootstrapping tablet: real 0.020s user 0.010s sys 0.005s
02:49:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.150291 26719 tablet_bootstrap.cc:654] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a: Neither blocks nor log segments found. Creating new log.
02:49:14.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.151870 26719 log.cc:824] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a: Log is configured to *not* fsync() on all Append() calls
02:49:14.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.155226 26716 tablet_bootstrap.cc:492] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea: No bootstrap required, opened a new log
02:49:14.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.155511 26719 tablet_bootstrap.cc:492] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a: No bootstrap required, opened a new log
02:49:14.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.155596 26716 ts_tablet_manager.cc:1397] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea: Time spent bootstrapping tablet: real 0.020s user 0.012s sys 0.004s
02:49:14.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.155804 26719 ts_tablet_manager.cc:1397] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a: Time spent bootstrapping tablet: real 0.011s user 0.007s sys 0.000s
02:49:14.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.158216 26717 raft_consensus.cc:348] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.158752 26717 raft_consensus.cc:374] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:14.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.158854 26717 raft_consensus.cc:729] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95899bff95504c12969daf319b305f5c, State: Initialized, Role: FOLLOWER
02:49:14.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.159631 26717 consensus_queue.cc:260] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.160936 26549 heartbeater.cc:503] Master 127.25.89.252:40643 was elected leader, sending a full tablet report...
02:49:14.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.161664 26717 ts_tablet_manager.cc:1428] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c: Time spent starting tablet: real 0.014s user 0.003s sys 0.012s
02:49:14.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.163056 26717 tablet_bootstrap.cc:492] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c: Bootstrap starting.
02:49:14.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.165146 26717 tablet_bootstrap.cc:654] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c: Neither blocks nor log segments found. Creating new log.
02:49:14.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.166340 26719 raft_consensus.cc:348] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.166882 26719 raft_consensus.cc:374] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:14.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.166985 26719 raft_consensus.cc:729] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 60171f79b6734d19a4b20f9b7f5ce54a, State: Initialized, Role: FOLLOWER
02:49:14.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.166774 26716 raft_consensus.cc:348] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.167359 26716 raft_consensus.cc:374] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:14.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.167608 26716 raft_consensus.cc:729] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f16375da86945e093b5a7cba9f06fea, State: Initialized, Role: FOLLOWER
02:49:14.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.168788 26716 consensus_queue.cc:260] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.167764 26719 consensus_queue.cc:260] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.175256 26691 heartbeater.cc:503] Master 127.25.89.252:40643 was elected leader, sending a full tablet report...
02:49:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.179386 26717 tablet_bootstrap.cc:492] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c: No bootstrap required, opened a new log
02:49:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.179745 26716 ts_tablet_manager.cc:1428] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea: Time spent starting tablet: real 0.024s user 0.018s sys 0.001s
02:49:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.179849 26717 ts_tablet_manager.cc:1397] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c: Time spent bootstrapping tablet: real 0.017s user 0.006s sys 0.004s
02:49:14.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.180938 26716 tablet_bootstrap.cc:492] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea: Bootstrap starting.
02:49:14.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.182876 26717 raft_consensus.cc:348] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } }
02:49:14.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.183346 26716 tablet_bootstrap.cc:654] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea: Neither blocks nor log segments found. Creating new log.
02:49:14.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.183597 26717 raft_consensus.cc:374] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:14.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.184146 26717 raft_consensus.cc:729] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95899bff95504c12969daf319b305f5c, State: Initialized, Role: FOLLOWER
02:49:14.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.184600 26717 consensus_queue.cc:260] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } }
02:49:14.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.185503 26717 ts_tablet_manager.cc:1428] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c: Time spent starting tablet: real 0.006s user 0.001s sys 0.001s
02:49:14.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.187973 26719 ts_tablet_manager.cc:1428] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a: Time spent starting tablet: real 0.032s user 0.014s sys 0.017s
02:49:14.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.192099 26719 tablet_bootstrap.cc:492] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a: Bootstrap starting.
02:49:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.195133 26719 tablet_bootstrap.cc:654] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a: Neither blocks nor log segments found. Creating new log.
02:49:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.198851 26719 tablet_bootstrap.cc:492] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a: No bootstrap required, opened a new log
02:49:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.199028 26719 ts_tablet_manager.cc:1397] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.000s
02:49:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.199721 26719 raft_consensus.cc:348] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } }
02:49:14.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.199921 26719 raft_consensus.cc:374] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:14.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.200013 26719 raft_consensus.cc:729] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 60171f79b6734d19a4b20f9b7f5ce54a, State: Initialized, Role: FOLLOWER
02:49:14.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.200265 26719 consensus_queue.cc:260] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } }
02:49:14.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.202142 26719 ts_tablet_manager.cc:1428] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
02:49:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.203773 26716 tablet_bootstrap.cc:492] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea: No bootstrap required, opened a new log
02:49:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.203960 26716 ts_tablet_manager.cc:1397] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea: Time spent bootstrapping tablet: real 0.023s user 0.007s sys 0.014s
02:49:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.204918 26716 raft_consensus.cc:348] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } }
02:49:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.205340 26716 raft_consensus.cc:374] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.205470 26716 raft_consensus.cc:729] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f16375da86945e093b5a7cba9f06fea, State: Initialized, Role: FOLLOWER
02:49:14.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.205741 26716 consensus_queue.cc:260] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } }
02:49:14.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.206370 26716 ts_tablet_manager.cc:1428] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
02:49:14.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:14.216328 26552 tablet.cc:2369] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:49:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.219938 26724 raft_consensus.cc:482] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:49:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.220301 26724 raft_consensus.cc:504] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } }
02:49:14.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.222151 26724 leader_election.cc:288] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0f16375da86945e093b5a7cba9f06fea (127.25.89.195:44665), 95899bff95504c12969daf319b305f5c (127.25.89.194:41711)
02:49:14.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.230424 26646 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddde257f75d543e99a5f14f9ffd7913c" candidate_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0f16375da86945e093b5a7cba9f06fea" is_pre_election: true
02:49:14.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.230425 26504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddde257f75d543e99a5f14f9ffd7913c" candidate_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95899bff95504c12969daf319b305f5c" is_pre_election: true
02:49:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.230918 26646 raft_consensus.cc:2454] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 60171f79b6734d19a4b20f9b7f5ce54a in term 0.
02:49:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.230938 26504 raft_consensus.cc:2454] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 60171f79b6734d19a4b20f9b7f5ce54a in term 0.
02:49:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.231561 26287 leader_election.cc:302] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 60171f79b6734d19a4b20f9b7f5ce54a, 95899bff95504c12969daf319b305f5c; no voters:
02:49:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.231948 26724 raft_consensus.cc:2781] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Leader pre-election won for term 1
02:49:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.232095 26724 raft_consensus.cc:482] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:49:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.232205 26724 raft_consensus.cc:3037] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Advancing to term 1
02:49:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.234287 26724 raft_consensus.cc:504] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } }
02:49:14.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.234876 26724 leader_election.cc:288] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [CANDIDATE]: Term 1 election: Requested vote from peers 0f16375da86945e093b5a7cba9f06fea (127.25.89.195:44665), 95899bff95504c12969daf319b305f5c (127.25.89.194:41711)
02:49:14.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.235481 26646 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddde257f75d543e99a5f14f9ffd7913c" candidate_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0f16375da86945e093b5a7cba9f06fea"
02:49:14.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.235481 26504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddde257f75d543e99a5f14f9ffd7913c" candidate_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95899bff95504c12969daf319b305f5c"
02:49:14.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.235730 26504 raft_consensus.cc:3037] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Advancing to term 1
02:49:14.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.235728 26646 raft_consensus.cc:3037] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Advancing to term 1
02:49:14.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:14.237517 26694 tablet.cc:2369] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:49:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.238906 26646 raft_consensus.cc:2454] T ddde257f75d543e99a5f14f9ffd7913c P 0f16375da86945e093b5a7cba9f06fea [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 60171f79b6734d19a4b20f9b7f5ce54a in term 1.
02:49:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.238906 26504 raft_consensus.cc:2454] T ddde257f75d543e99a5f14f9ffd7913c P 95899bff95504c12969daf319b305f5c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 60171f79b6734d19a4b20f9b7f5ce54a in term 1.
02:49:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.239447 26287 leader_election.cc:302] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 60171f79b6734d19a4b20f9b7f5ce54a, 95899bff95504c12969daf319b305f5c; no voters:
02:49:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.239792 26724 raft_consensus.cc:2781] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [term 1 FOLLOWER]: Leader election won for term 1
02:49:14.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.240253 26724 raft_consensus.cc:686] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [term 1 LEADER]: Becoming Leader. State: Replica: 60171f79b6734d19a4b20f9b7f5ce54a, State: Running, Role: LEADER
02:49:14.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.240770 26724 consensus_queue.cc:237] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } }
02:49:14.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.246002 26162 catalog_manager.cc:5495] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a reported cstate change: term changed from 0 to 1, leader changed from <none> to 60171f79b6734d19a4b20f9b7f5ce54a (127.25.89.193). New cstate: current_term: 1 leader_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } health_report { overall_health: HEALTHY } } }
02:49:14.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.367964 26725 raft_consensus.cc:482] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:49:14.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.368211 26725 raft_consensus.cc:504] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.369971 26725 leader_election.cc:288] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 60171f79b6734d19a4b20f9b7f5ce54a (127.25.89.193:35885), 95899bff95504c12969daf319b305f5c (127.25.89.194:41711)
02:49:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.376695 26504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0d88dd7fe3274724aee1670794deb4ea" candidate_uuid: "0f16375da86945e093b5a7cba9f06fea" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95899bff95504c12969daf319b305f5c" is_pre_election: true
02:49:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.376971 26504 raft_consensus.cc:2454] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0f16375da86945e093b5a7cba9f06fea in term 0.
02:49:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.376915 26363 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0d88dd7fe3274724aee1670794deb4ea" candidate_uuid: "0f16375da86945e093b5a7cba9f06fea" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" is_pre_election: true
02:49:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.377381 26363 raft_consensus.cc:2454] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0f16375da86945e093b5a7cba9f06fea in term 0.
02:49:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.377439 26573 leader_election.cc:302] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0f16375da86945e093b5a7cba9f06fea, 95899bff95504c12969daf319b305f5c; no voters:
02:49:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.377898 26725 raft_consensus.cc:2781] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Leader pre-election won for term 1
02:49:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.378005 26725 raft_consensus.cc:482] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:49:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.378070 26725 raft_consensus.cc:3037] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Advancing to term 1
02:49:14.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.380031 26725 raft_consensus.cc:504] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.380682 26725 leader_election.cc:288] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [CANDIDATE]: Term 1 election: Requested vote from peers 60171f79b6734d19a4b20f9b7f5ce54a (127.25.89.193:35885), 95899bff95504c12969daf319b305f5c (127.25.89.194:41711)
02:49:14.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.381275 26363 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0d88dd7fe3274724aee1670794deb4ea" candidate_uuid: "0f16375da86945e093b5a7cba9f06fea" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "60171f79b6734d19a4b20f9b7f5ce54a"
02:49:14.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.381271 26504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0d88dd7fe3274724aee1670794deb4ea" candidate_uuid: "0f16375da86945e093b5a7cba9f06fea" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95899bff95504c12969daf319b305f5c"
02:49:14.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.381471 26363 raft_consensus.cc:3037] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Advancing to term 1
02:49:14.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.381489 26504 raft_consensus.cc:3037] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Advancing to term 1
02:49:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.383531 26504 raft_consensus.cc:2454] T 0d88dd7fe3274724aee1670794deb4ea P 95899bff95504c12969daf319b305f5c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f16375da86945e093b5a7cba9f06fea in term 1.
02:49:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.383626 26363 raft_consensus.cc:2454] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f16375da86945e093b5a7cba9f06fea in term 1.
02:49:14.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.383953 26573 leader_election.cc:302] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0f16375da86945e093b5a7cba9f06fea, 95899bff95504c12969daf319b305f5c; no voters:
02:49:14.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.384230 26725 raft_consensus.cc:2781] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [term 1 FOLLOWER]: Leader election won for term 1
02:49:14.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.384610 26725 raft_consensus.cc:686] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [term 1 LEADER]: Becoming Leader. State: Replica: 0f16375da86945e093b5a7cba9f06fea, State: Running, Role: LEADER
02:49:14.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.385116 26725 consensus_queue.cc:237] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.390235 26162 catalog_manager.cc:5495] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea reported cstate change: term changed from 0 to 1, leader changed from <none> to 0f16375da86945e093b5a7cba9f06fea (127.25.89.195). New cstate: current_term: 1 leader_uuid: "0f16375da86945e093b5a7cba9f06fea" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } health_report { overall_health: HEALTHY } } }
02:49:14.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:14.433451 26411 tablet.cc:2369] T 0d88dd7fe3274724aee1670794deb4ea P 60171f79b6734d19a4b20f9b7f5ce54a: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:49:14.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.722358 26724 consensus_queue.cc:1035] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:49:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.731499 26724 consensus_queue.cc:1035] T ddde257f75d543e99a5f14f9ffd7913c P 60171f79b6734d19a4b20f9b7f5ce54a [LEADER]: Connected to new peer: Peer: permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:49:14.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.850540 26725 consensus_queue.cc:1035] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [LEADER]: Connected to new peer: Peer: permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:49:14.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.861159 26733 consensus_queue.cc:1035] T 0d88dd7fe3274724aee1670794deb4ea P 0f16375da86945e093b5a7cba9f06fea [LEADER]: Connected to new peer: Peer: permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.873277 26163 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35462:
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
02:49:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
02:49:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:14.874250 26163 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:49:14.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.887578 26333 tablet_service.cc:1467] Processing CreateTablet for tablet a763c8177d6548ebaa39384edca1a3b9 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=6e978802443d4b1fb2b61b38fbaac2bb]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
02:49:14.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.888619 26333 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a763c8177d6548ebaa39384edca1a3b9. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:14.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.888648 26477 tablet_service.cc:1467] Processing CreateTablet for tablet a763c8177d6548ebaa39384edca1a3b9 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=6e978802443d4b1fb2b61b38fbaac2bb]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
02:49:14.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.889657 26477 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a763c8177d6548ebaa39384edca1a3b9. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:14.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.895282 26618 tablet_service.cc:1467] Processing CreateTablet for tablet a763c8177d6548ebaa39384edca1a3b9 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=6e978802443d4b1fb2b61b38fbaac2bb]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
02:49:14.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.896531 26618 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a763c8177d6548ebaa39384edca1a3b9. 1 dirs total, 0 dirs full, 0 dirs failed
02:49:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.900987 26756 tablet_bootstrap.cc:492] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a: Bootstrap starting.
02:49:14.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.903287 26756 tablet_bootstrap.cc:654] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a: Neither blocks nor log segments found. Creating new log.
02:49:14.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.904521 26755 tablet_bootstrap.cc:492] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: Bootstrap starting.
02:49:14.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.905805 26758 tablet_bootstrap.cc:492] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea: Bootstrap starting.
02:49:14.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.906219 26755 tablet_bootstrap.cc:654] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: Neither blocks nor log segments found. Creating new log.
02:49:14.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.906257 26756 tablet_bootstrap.cc:492] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a: No bootstrap required, opened a new log
02:49:14.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.906404 26756 ts_tablet_manager.cc:1397] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.003s
02:49:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.907235 26756 raft_consensus.cc:348] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.907451 26758 tablet_bootstrap.cc:654] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea: Neither blocks nor log segments found. Creating new log.
02:49:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.907516 26756 raft_consensus.cc:374] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.907603 26756 raft_consensus.cc:729] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 60171f79b6734d19a4b20f9b7f5ce54a, State: Initialized, Role: FOLLOWER
02:49:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.907863 26756 consensus_queue.cc:260] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.908419 26724 tablet_replica.cc:439] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } }
02:49:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.908777 26724 tablet_replica.cc:442] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a: This TxnStatusTablet replica's current role is: FOLLOWER
02:49:14.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.908447 26756 ts_tablet_manager.cc:1428] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
02:49:14.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.915418 26755 tablet_bootstrap.cc:492] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: No bootstrap required, opened a new log
02:49:14.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.915782 26755 ts_tablet_manager.cc:1397] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.003s
02:49:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.916741 26755 raft_consensus.cc:348] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.917002 26755 raft_consensus.cc:374] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.917086 26755 raft_consensus.cc:729] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95899bff95504c12969daf319b305f5c, State: Initialized, Role: FOLLOWER
02:49:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.917371 26755 consensus_queue.cc:260] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.917894 26739 tablet_replica.cc:439] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } }
02:49:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.918272 26739 tablet_replica.cc:442] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: This TxnStatusTablet replica's current role is: FOLLOWER
02:49:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.920877 26755 ts_tablet_manager.cc:1428] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
02:49:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.921855 26758 tablet_bootstrap.cc:492] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea: No bootstrap required, opened a new log
02:49:14.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.922011 26758 ts_tablet_manager.cc:1397] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea: Time spent bootstrapping tablet: real 0.016s user 0.000s sys 0.015s
02:49:14.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.922816 26758 raft_consensus.cc:348] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.923049 26758 raft_consensus.cc:374] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:49:14.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.923118 26758 raft_consensus.cc:729] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f16375da86945e093b5a7cba9f06fea, State: Initialized, Role: FOLLOWER
02:49:14.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.923321 26758 consensus_queue.cc:260] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:14.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.923722 26733 tablet_replica.cc:439] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } }
02:49:14.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.924037 26733 tablet_replica.cc:442] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea: This TxnStatusTablet replica's current role is: FOLLOWER
02:49:14.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:14.925753 26758 ts_tablet_manager.cc:1428] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
02:49:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.008765 26739 raft_consensus.cc:482] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:49:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.009028 26739 raft_consensus.cc:504] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:15.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.010804 26739 leader_election.cc:288] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 60171f79b6734d19a4b20f9b7f5ce54a (127.25.89.193:35885), 0f16375da86945e093b5a7cba9f06fea (127.25.89.195:44665)
02:49:15.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.017899 26363 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a763c8177d6548ebaa39384edca1a3b9" candidate_uuid: "95899bff95504c12969daf319b305f5c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" is_pre_election: true
02:49:15.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.018210 26363 raft_consensus.cc:2454] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 95899bff95504c12969daf319b305f5c in term 0.
02:49:15.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.018144 26646 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a763c8177d6548ebaa39384edca1a3b9" candidate_uuid: "95899bff95504c12969daf319b305f5c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0f16375da86945e093b5a7cba9f06fea" is_pre_election: true
02:49:15.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.018393 26646 raft_consensus.cc:2454] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 95899bff95504c12969daf319b305f5c in term 0.
02:49:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.018780 26428 leader_election.cc:302] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 60171f79b6734d19a4b20f9b7f5ce54a, 95899bff95504c12969daf319b305f5c; no voters:
02:49:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.019223 26739 raft_consensus.cc:2781] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Leader pre-election won for term 1
02:49:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.019361 26739 raft_consensus.cc:482] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:49:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.019421 26739 raft_consensus.cc:3037] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [term 0 FOLLOWER]: Advancing to term 1
02:49:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.021457 26739 raft_consensus.cc:504] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:15.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.022106 26739 leader_election.cc:288] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [CANDIDATE]: Term 1 election: Requested vote from peers 60171f79b6734d19a4b20f9b7f5ce54a (127.25.89.193:35885), 0f16375da86945e093b5a7cba9f06fea (127.25.89.195:44665)
02:49:15.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.022706 26363 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a763c8177d6548ebaa39384edca1a3b9" candidate_uuid: "95899bff95504c12969daf319b305f5c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "60171f79b6734d19a4b20f9b7f5ce54a"
02:49:15.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.022720 26646 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a763c8177d6548ebaa39384edca1a3b9" candidate_uuid: "95899bff95504c12969daf319b305f5c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0f16375da86945e093b5a7cba9f06fea"
02:49:15.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.022953 26646 raft_consensus.cc:3037] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea [term 0 FOLLOWER]: Advancing to term 1
02:49:15.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.022953 26363 raft_consensus.cc:3037] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a [term 0 FOLLOWER]: Advancing to term 1
02:49:15.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.025861 26646 raft_consensus.cc:2454] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 95899bff95504c12969daf319b305f5c in term 1.
02:49:15.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.025861 26363 raft_consensus.cc:2454] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 95899bff95504c12969daf319b305f5c in term 1.
02:49:15.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.026466 26430 leader_election.cc:302] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 0f16375da86945e093b5a7cba9f06fea, 60171f79b6734d19a4b20f9b7f5ce54a, 95899bff95504c12969daf319b305f5c; no voters:
02:49:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.026899 26739 raft_consensus.cc:2781] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [term 1 FOLLOWER]: Leader election won for term 1
02:49:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.027428 26739 raft_consensus.cc:686] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [term 1 LEADER]: Becoming Leader. State: Replica: 95899bff95504c12969daf319b305f5c, State: Running, Role: LEADER
02:49:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.027936 26739 consensus_queue.cc:237] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } }
02:49:15.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.030171 26765 tablet_replica.cc:439] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: TxnStatusTablet state changed. Reason: New leader 95899bff95504c12969daf319b305f5c. Latest consensus state: current_term: 1 leader_uuid: "95899bff95504c12969daf319b305f5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } }
02:49:15.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.030833 26765 tablet_replica.cc:442] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: This TxnStatusTablet replica's current role is: LEADER
02:49:15.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.033313 26767 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
02:49:15.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.033752 26767 txn_status_manager.cc:929] Loading transaction status metadata into memory...
02:49:15.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.034307 26163 catalog_manager.cc:5495] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c reported cstate change: term changed from 0 to 1, leader changed from <none> to 95899bff95504c12969daf319b305f5c (127.25.89.194). New cstate: current_term: 1 leader_uuid: "95899bff95504c12969daf319b305f5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } health_report { overall_health: UNKNOWN } } }
02:49:15.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.078611 26646 raft_consensus.cc:1261] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea [term 1 FOLLOWER]: Refusing update from remote peer 95899bff95504c12969daf319b305f5c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:49:15.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.078606 26363 raft_consensus.cc:1261] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a [term 1 FOLLOWER]: Refusing update from remote peer 95899bff95504c12969daf319b305f5c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:49:15.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.079494 26739 consensus_queue.cc:1035] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [LEADER]: Connected to new peer: Peer: permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:49:15.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.079883 26765 consensus_queue.cc:1035] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:49:15.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.096828 26733 tablet_replica.cc:439] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea: TxnStatusTablet state changed. Reason: New leader 95899bff95504c12969daf319b305f5c. Latest consensus state: current_term: 1 leader_uuid: "95899bff95504c12969daf319b305f5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } }
02:49:15.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.097229 26733 tablet_replica.cc:442] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea: This TxnStatusTablet replica's current role is: FOLLOWER
02:49:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.100646 26733 tablet_replica.cc:439] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "95899bff95504c12969daf319b305f5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } }
02:49:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.100926 26733 tablet_replica.cc:442] T a763c8177d6548ebaa39384edca1a3b9 P 0f16375da86945e093b5a7cba9f06fea: This TxnStatusTablet replica's current role is: FOLLOWER
02:49:15.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.101137 26741 mvcc.cc:205] Tried to move back new op lower bound from 7298405806392143872 to 7298405806199115776. Current Snapshot: MvccSnapshot[applied={T|T < 7298405806392143872}]
02:49:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.111899 26724 tablet_replica.cc:439] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a: TxnStatusTablet state changed. Reason: New leader 95899bff95504c12969daf319b305f5c. Latest consensus state: current_term: 1 leader_uuid: "95899bff95504c12969daf319b305f5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } }
02:49:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.112257 26724 tablet_replica.cc:442] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a: This TxnStatusTablet replica's current role is: FOLLOWER
02:49:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.117962 26765 tablet_replica.cc:439] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "95899bff95504c12969daf319b305f5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } }
02:49:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.118330 26765 tablet_replica.cc:442] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: This TxnStatusTablet replica's current role is: LEADER
02:49:15.122 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
02:49:15.125 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 6430e55dd2ab4e378ff92a4c488c280f
02:49:15.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.120111 26765 tablet_replica.cc:439] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "95899bff95504c12969daf319b305f5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } }
02:49:15.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.120474 26765 tablet_replica.cc:442] T a763c8177d6548ebaa39384edca1a3b9 P 95899bff95504c12969daf319b305f5c: This TxnStatusTablet replica's current role is: LEADER
02:49:15.127 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet ddde257f75d543e99a5f14f9ffd7913c for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
02:49:15.127 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 0d88dd7fe3274724aee1670794deb4ea for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
02:49:15.128 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=ddde257f75d543e99a5f14f9ffd7913c@[0f16375da86945e093b5a7cba9f06fea(127.25.89.195:44665),60171f79b6734d19a4b20f9b7f5ce54a(127.25.89.193:35885)[L],95899bff95504c12969daf319b305f5c(127.25.89.194:41711)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=0d88dd7fe3274724aee1670794deb4ea@[0f16375da86945e093b5a7cba9f06fea(127.25.89.195:44665)[L],60171f79b6734d19a4b20f9b7f5ce54a(127.25.89.193:35885),95899bff95504c12969daf319b305f5c(127.25.89.194:41711)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
02:49:15.128 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.25.89.252:40643, [2ms] Sub RPC GetTableLocations: received response from server master-127.25.89.252:40643: OK, deferred=null) after lookup
02:49:15.128 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 60171f79b6734d19a4b20f9b7f5ce54a(127.25.89.193:35885)] connecting to peer
02:49:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.127876 26724 tablet_replica.cc:439] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "95899bff95504c12969daf319b305f5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "60171f79b6734d19a4b20f9b7f5ce54a" member_type: VOTER last_known_addr { host: "127.25.89.193" port: 35885 } } peers { permanent_uuid: "95899bff95504c12969daf319b305f5c" member_type: VOTER last_known_addr { host: "127.25.89.194" port: 41711 } } peers { permanent_uuid: "0f16375da86945e093b5a7cba9f06fea" member_type: VOTER last_known_addr { host: "127.25.89.195" port: 44665 } } }
02:49:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.128192 26724 tablet_replica.cc:442] T a763c8177d6548ebaa39384edca1a3b9 P 60171f79b6734d19a4b20f9b7f5ce54a: This TxnStatusTablet replica's current role is: FOLLOWER
02:49:15.134 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 60171f79b6734d19a4b20f9b7f5ce54a(127.25.89.193:35885)] Successfully connected to peer
02:49:15.150 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x3c355b32, L:/127.0.0.1:45972 - R:/127.25.89.193:35885] using TOKEN/null
02:49:15.206 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.89.252:40643
02:49:15.207 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:49:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.208913 25959 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu pid 26129
02:49:15.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.210327 26203 generic_service.cc:196] Checking for leaks (request via RPC)
02:49:15.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.333506 25959 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu with pid 26129
02:49:15.351 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.25.89.252:40643(127.25.89.252:40643)] handling channelInactive
02:49:15.351 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state READY due to: connection closed
02:49:15.353 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:49:15.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:15.491209 26691 heartbeater.cc:644] Failed to heartbeat to 127.25.89.252:40643 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:15.630 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.632 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.25.89.252:40643(127.25.89.252:40643) for tablet Kudu Master: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.632972 26784 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 57d79a0a4f074da4b31478e6fdaa1954)
02:49:15.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.633141 26785 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 57d79a0a4f074da4b31478e6fdaa1954)
02:49:15.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.633231 26784 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:15.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.633395 26785 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:15.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.634654 26785 leader_election.cc:288] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3af63bec99064cbbb230e2f985fd1a34 (127.25.89.254:40383), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.635192 26091 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3af63bec99064cbbb230e2f985fd1a34" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "9c0f8ffb53444dc5bddc487da2466d59" is_pre_election: true
02:49:15.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.635483 26091 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3af63bec99064cbbb230e2f985fd1a34 in term 1.
02:49:15.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.635885 26784 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:15.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.636085 26010 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9c0f8ffb53444dc5bddc487da2466d59" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "3af63bec99064cbbb230e2f985fd1a34" is_pre_election: true
02:49:15.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.636346 26010 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c0f8ffb53444dc5bddc487da2466d59 in term 1.
02:49:15.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.637001 25985 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34, 9c0f8ffb53444dc5bddc487da2466d59; no voters:
02:49:15.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.637436 26784 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:49:15.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.637559 26784 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 57d79a0a4f074da4b31478e6fdaa1954)
02:49:15.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.637705 26784 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 1 FOLLOWER]: Advancing to term 2
02:49:15.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.637770 26069 leader_election.cc:302] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34, 9c0f8ffb53444dc5bddc487da2466d59; no voters:
02:49:15.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.638362 26785 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:49:15.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.638478 26785 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 57d79a0a4f074da4b31478e6fdaa1954)
02:49:15.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.638563 26785 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 1 FOLLOWER]: Advancing to term 2
02:49:15.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:15.638561 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:15.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:15.640586 26067 leader_election.cc:334] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.640834 26784 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.641587 26784 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 2 election: Requested vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.641571 26785 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.642056 26091 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3af63bec99064cbbb230e2f985fd1a34" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "9c0f8ffb53444dc5bddc487da2466d59"
02:49:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.642330 26091 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 3af63bec99064cbbb230e2f985fd1a34 in current term 2: Already voted for candidate 9c0f8ffb53444dc5bddc487da2466d59 in this term.
02:49:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.643179 26785 leader_election.cc:288] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 2 election: Requested vote from peers 3af63bec99064cbbb230e2f985fd1a34 (127.25.89.254:40383), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:15.643190 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.643362 25986 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.643334 26010 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9c0f8ffb53444dc5bddc487da2466d59" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "3af63bec99064cbbb230e2f985fd1a34"
02:49:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.643662 26010 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 9c0f8ffb53444dc5bddc487da2466d59 in current term 2: Already voted for candidate 3af63bec99064cbbb230e2f985fd1a34 in this term.
02:49:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.643841 26784 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
02:49:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:15.644412 26067 leader_election.cc:334] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.644518 26067 leader_election.cc:302] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9c0f8ffb53444dc5bddc487da2466d59; no voters: 3af63bec99064cbbb230e2f985fd1a34, 57d79a0a4f074da4b31478e6fdaa1954
02:49:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.644796 26785 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
02:49:15.647 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.648 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.648 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.648 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.649 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.667 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.668 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.668 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.669 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.687 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.707 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.708 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.710 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.727 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.768 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.847 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.852 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.25.89.252:40643, [5ms] received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [5ms] delaying RPC due to: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [21ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [41ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [42ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [62ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [62ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [82ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [83ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [102ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [102ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [142ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [143ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [222ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [223ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, deferred=Deferred@565846645(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1087663577@d828b8b -> (continuation of Deferred@1721874544 after retry RPC after error@1490734341) -> (continuation of Deferred@679732705 after retry RPC after error@826604969) -> (continuation of Deferred@1523429833 after retry RPC after error@1425141104) -> (continuation of Deferred@1676326626 after retry RPC after error@1976936067) -> (continuation of Deferred@1117848073 after retry RPC after error@425782592) -> (continuation of Deferred@789251368 after retry RPC after error@2008174580), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1515517835@3dc9aa41 -> (continuation of Deferred@1721874544 after retry RPC after error@1490734341) -> (continuation of Deferred@679732705 after retry RPC after error@826604969) -> (continuation of Deferred@1523429833 after retry RPC after error@1425141104) -> (continuation of Deferred@1676326626 after retry RPC after error@1976936067) -> (continuation of Deferred@1117848073 after retry RPC after error@425782592) -> (continuation of Deferred@789251368 after retry RPC after error@2008174580)))
02:49:15.852 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] sending RPC to server master-127.25.89.252:40643, [5ms] received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [5ms] delaying RPC due to: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [21ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [41ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [42ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [62ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [62ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [82ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [83ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [102ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [102ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [142ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [143ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [222ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [223ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, deferred=Deferred@565846645(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1087663577@d828b8b -> (continuation of Deferred@1721874544 after retry RPC after error@1490734341) -> (continuation of Deferred@679732705 after retry RPC after error@826604969) -> (continuation of Deferred@1523429833 after retry RPC after error@1425141104) -> (continuation of Deferred@1676326626 after retry RPC after error@1976936067) -> (continuation of Deferred@1117848073 after retry RPC after error@425782592) -> (continuation of Deferred@789251368 after retry RPC after error@2008174580), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/1515517835@3dc9aa41 -> (continuation of Deferred@1721874544 after retry RPC after error@1490734341) -> (continuation of Deferred@679732705 after retry RPC after error@826604969) -> (continuation of Deferred@1523429833 after retry RPC after error@1425141104) -> (continuation of Deferred@1676326626 after retry RPC after error@1976936067) -> (continuation of Deferred@1117848073 after retry RPC after error@425782592) -> (continuation of Deferred@789251368 after retry RPC after error@2008174580)))
02:49:15.867 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.888 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.908 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.910 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.928 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.932 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.933 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.935 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.968 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:15.970 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.971 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:15.971 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:15.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.995226 26785 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:49:15.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.995469 26785 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.996166 26785 leader_election.cc:288] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 3af63bec99064cbbb230e2f985fd1a34 (127.25.89.254:40383), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:15.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.996779 26010 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9c0f8ffb53444dc5bddc487da2466d59" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "3af63bec99064cbbb230e2f985fd1a34" is_pre_election: true
02:49:15.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.997053 26010 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c0f8ffb53444dc5bddc487da2466d59 in term 2.
02:49:15.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.997509 26069 leader_election.cc:302] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34, 9c0f8ffb53444dc5bddc487da2466d59; no voters:
02:49:15.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.997826 26785 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 2 FOLLOWER]: Leader pre-election won for term 3
02:49:15.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:15.997848 26067 leader_election.cc:334] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:15.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.997925 26785 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:49:15.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.998014 26785 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 2 FOLLOWER]: Advancing to term 3
02:49:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:15.999902 26785 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.000491 26785 leader_election.cc:288] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 3 election: Requested vote from peers 3af63bec99064cbbb230e2f985fd1a34 (127.25.89.254:40383), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:16.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.001183 26010 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9c0f8ffb53444dc5bddc487da2466d59" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "3af63bec99064cbbb230e2f985fd1a34"
02:49:16.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.001467 26010 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 2 FOLLOWER]: Advancing to term 3
02:49:16.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:16.002533 26067 leader_election.cc:334] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:16.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.003746 26010 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c0f8ffb53444dc5bddc487da2466d59 in term 3.
02:49:16.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.004217 26069 leader_election.cc:302] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34, 9c0f8ffb53444dc5bddc487da2466d59; no voters: 57d79a0a4f074da4b31478e6fdaa1954
02:49:16.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.004522 26785 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 3 FOLLOWER]: Leader election won for term 3
02:49:16.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.005000 26785 raft_consensus.cc:686] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [term 3 LEADER]: Becoming Leader. State: Replica: 9c0f8ffb53444dc5bddc487da2466d59, State: Running, Role: LEADER
02:49:16.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.005486 26785 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:16.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.006970 26791 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9c0f8ffb53444dc5bddc487da2466d59. Latest consensus state: current_term: 3 leader_uuid: "9c0f8ffb53444dc5bddc487da2466d59" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:16.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.007308 26791 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [sys.catalog]: This master's current role is: LEADER
02:49:16.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.008031 26792 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
02:49:16.008 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:16.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:16.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:16.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:16.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.011008 26792 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback [id=6430e55dd2ab4e378ff92a4c488c280f]
02:49:16.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.011461 26792 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=6e978802443d4b1fb2b61b38fbaac2bb]
02:49:16.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.013152 26792 tablet_loader.cc:96] loaded metadata for tablet 0d88dd7fe3274724aee1670794deb4ea (table txn_manager_fallback [id=6430e55dd2ab4e378ff92a4c488c280f])
02:49:16.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.013583 26792 tablet_loader.cc:96] loaded metadata for tablet a763c8177d6548ebaa39384edca1a3b9 (table kudu_system.kudu_transactions [id=6e978802443d4b1fb2b61b38fbaac2bb])
02:49:16.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.013898 26792 tablet_loader.cc:96] loaded metadata for tablet ddde257f75d543e99a5f14f9ffd7913c (table txn_manager_fallback [id=6430e55dd2ab4e378ff92a4c488c280f])
02:49:16.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.014127 26792 catalog_manager.cc:1484] Initializing Kudu cluster ID...
02:49:16.015 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:16.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.015241 26792 catalog_manager.cc:1259] Loaded cluster ID: 560d1240bf214b3398c335ec5b2de248
02:49:16.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.015347 26792 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
02:49:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.017045 26792 catalog_manager.cc:1504] Loading token signing keys...
02:49:16.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.018276 26792 catalog_manager.cc:5879] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59: Loaded TSK: 0
02:49:16.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.019177 26792 catalog_manager.cc:1514] Initializing in-progress tserver states...
02:49:16.068 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:16.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:16.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:16.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:16.071 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:49:16.072 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=204), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [3ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [3ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [42ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [43ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [65ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [66ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [68ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [68ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [103ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [104ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [142ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [143ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [147ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [148ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [202ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [203ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [204ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, deferred=Deferred@1381081543(state=PENDING, result=null, callback=(continuation of Deferred@504138575 after retry RPC after error@413216347) -> (continuation of Deferred@497945330 after retry RPC after error@1110368025) -> (continuation of Deferred@948073184 after retry RPC after error@700465245) -> (continuation of Deferred@1154754061 after retry RPC after error@1882652569) -> (continuation of Deferred@1535673869 after retry RPC after error@1885464457) -> (continuation of Deferred@367018950 after retry RPC after error@1382923476), errback=(continuation of Deferred@504138575 after retry RPC after error@413216347) -> (continuation of Deferred@497945330 after retry RPC after error@1110368025) -> (continuation of Deferred@948073184 after retry RPC after error@700465245) -> (continuation of Deferred@1154754061 after retry RPC after error@1882652569) -> (continuation of Deferred@1535673869 after retry RPC after error@1885464457) -> (continuation of Deferred@367018950 after retry RPC after error@1382923476))) after lookup
02:49:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.106309 26010 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Refusing update from remote peer 9c0f8ffb53444dc5bddc487da2466d59: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
02:49:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:16.106953 26067 consensus_peers.cc:487] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 -> Peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Couldn't send request to peer 57d79a0a4f074da4b31478e6fdaa1954. Status: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:49:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.107118 26791 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
02:49:16.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:16.118767 26112 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:56298 (request call id 15) took 45 ms (client timeout 45 ms). Trace:
02:49:16.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:16.119065 26112 rpcz_store.cc:269] 0619 02:49:16.073368 (+ 0us) service_pool.cc:170] Inserting onto call queue
02:49:16.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0619 02:49:16.073457 (+ 89us) service_pool.cc:227] Handling call
02:49:16.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0619 02:49:16.118630 (+ 45173us) inbound_call.cc:173] Queueing success response
02:49:16.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":200,"init.queue_time_us":549,"thread_start_us":1834,"threads_started":5}
02:49:16.130 [DEBUG - pool-46-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=262), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [3ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [3ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [42ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [43ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [65ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [66ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [68ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [68ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [103ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [104ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [142ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [143ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [147ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [148ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [202ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [203ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [204ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: OK, [205ms] sending RPC to server master-127.25.89.253:46647, deferred=Deferred@1381081543(state=PENDING, result=null, callback=(continuation of Deferred@504138575 after retry RPC after error@413216347) -> (continuation of Deferred@497945330 after retry RPC after error@1110368025) -> (continuation of Deferred@948073184 after retry RPC after error@700465245) -> (continuation of Deferred@1154754061 after retry RPC after error@1882652569) -> (continuation of Deferred@1535673869 after retry RPC after error@1885464457) -> (continuation of Deferred@367018950 after retry RPC after error@1382923476) -> (continuation of Deferred@1174224802 after retry RPC@1451910538), errback=(continuation of Deferred@504138575 after retry RPC after error@413216347) -> (continuation of Deferred@497945330 after retry RPC after error@1110368025) -> (continuation of Deferred@948073184 after retry RPC after error@700465245) -> (continuation of Deferred@1154754061 after retry RPC after error@1882652569) -> (continuation of Deferred@1535673869 after retry RPC after error@1885464457) -> (continuation of Deferred@367018950 after retry RPC after error@1382923476) -> (continuation of Deferred@1174224802 after retry RPC@1451910538)))
02:49:16.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/monotime.cc:332:28: runtime error: signed integer overflow: 1488538798364 + 9223370548315984771 cannot be represented in type 'long'
02:49:16.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.160290 26784 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9c0f8ffb53444dc5bddc487da2466d59. Latest consensus state: current_term: 3 leader_uuid: "9c0f8ffb53444dc5bddc487da2466d59" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:16.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.160622 26784 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [sys.catalog]: This master's current role is: FOLLOWER
02:49:16.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:16.165110 26549 heartbeater.cc:644] Failed to heartbeat to 127.25.89.252:40643 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.167709 26784 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "9c0f8ffb53444dc5bddc487da2466d59" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.168017 26784 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [sys.catalog]: This master's current role is: FOLLOWER
02:49:16.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.168421 26434 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.021s from last keepalive heartbeat (effective timeout is 1.000s)
02:49:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:16.170502 26482 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.25.89.252:40643)
02:49:16.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.191924 26785 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "9c0f8ffb53444dc5bddc487da2466d59" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } } }
02:49:16.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.193264 26785 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 [sys.catalog]: This master's current role is: LEADER
02:49:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:16.210111 26408 heartbeater.cc:644] Failed to heartbeat to 127.25.89.252:40643 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:16.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:16.206012 26081 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56328:
02:49:16.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
02:49:16.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
02:49:16.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:16.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
02:49:16.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
02:49:16.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:49:16.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
02:49:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
02:49:16.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:16.219615 26081 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
02:49:16.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:16.222810 26807 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
02:49:16.263 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 95899bff95504c12969daf319b305f5c (127.25.89.194:41711): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1781837355 last_transition_timestamp: 1781837356
02:49:16.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:16.574932 26067 consensus_peers.cc:487] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 -> Peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Couldn't send request to peer 57d79a0a4f074da4b31478e6fdaa1954. Status: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:49:16.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:16.654515 26067 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111) [suppressed 14 similar messages]
02:49:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:17.041250 26067 consensus_peers.cc:487] T 00000000000000000000000000000000 P 9c0f8ffb53444dc5bddc487da2466d59 -> Peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Couldn't send request to peer 57d79a0a4f074da4b31478e6fdaa1954. Status: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
02:49:17.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.119545 25970 master_runner.cc:186] Error getting master registration for 127.25.89.252:40643: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111),
02:49:17.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.119778 25970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:49:17.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7f1680590538 in kudu::operator+(kudu::MonoTime const&, kudu::MonoDelta const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/monotime.cc:332:28
02:49:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.151814 26551 heartbeater.cc:503] Master 127.25.89.253:46647 was elected leader, sending a full tablet report...
02:49:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0x9dd255 in kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:99:38
02:49:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.200171 26410 heartbeater.cc:503] Master 127.25.89.253:46647 was elected leader, sending a full tablet report...
02:49:17.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7f1693600be2 in kudu::Promise<kudu::Status>::WaitFor(kudu::MonoDelta const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/promise.h:57:16
02:49:17.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7f16935f17d4 in kudu::master::Master::WaitForTxnManagerInit(kudu::MonoTime) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:458:48
02:49:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7f169374616b in kudu::transactions::TxnManager::CheckInitialized(kudu::MonoTime const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/txn_manager.cc:283:21
02:49:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7f1693746d5b in kudu::transactions::TxnManager::KeepTransactionAlive(long, std::string const&, kudu::MonoTime const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/txn_manager.cc:226:3
02:49:17.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7f169374b9cf in kudu::transactions::TxnManagerServiceImpl::KeepTransactionAlive(kudu::transactions::KeepTransactionAliveRequestPB const*, kudu::transactions::KeepTransactionAliveResponsePB*, kudu::rpc::RpcContext*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/txn_manager_service.cc:159:42
02:49:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7f168d7fccc6 in kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/build/asan/src/kudu/master/txn_manager.service.cc:271:13
02:49:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7f168d7fc9f5 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/8/bits/std_function.h:297:2
02:49:17.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7f168716d9be in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/8/bits/std_function.h:687:14
02:49:17.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7f168716c79e in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_if.cc:137:3
02:49:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7f1687172b25 in kudu::rpc::ServicePool::RunThread() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_pool.cc:231:15
02:49:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7f168717354f in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_pool.cc:92:5
02:49:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7f1687173378 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/8/bits/std_function.h:297:2
02:49:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0x9edb7c in std::function<void ()>::operator()() const ../../../include/c++/8/bits/std_function.h:687:14
02:49:17.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7f1680655eae in kudu::Thread::SuperviseThread(void*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694:3
02:49:17.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7f168392f6da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
02:49:17.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7f167c8c561e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12161e)
02:49:17.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:49:17.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/monotime.cc:332:28 in
02:49:17.274 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.25.89.253:46647(127.25.89.253:46647)] handling channelInactive
02:49:17.274 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state READY due to: connection closed
02:49:17.274 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.25.89.253:46647(127.25.89.253:46647) for tablet Kudu Master: connection closed
02:49:17.288 [WARN - pool-46-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
02:49:17.353 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:17.354 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:17.355 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.355 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.355 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.355 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.355 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.355 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.357 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:17.367 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:17.368 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:17.368 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.368 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:17.388 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:17.388 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:17.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:17.408 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:17.408 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:17.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:17.428 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:17.429 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:17.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:17.467 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:17.468 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:17.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.492483 26693 heartbeater.cc:503] Master 127.25.89.253:46647 was elected leader, sending a full tablet report...
02:49:17.527 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:17.528 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:17.528 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.528 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.547142 26835 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:49:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.547387 26835 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.548107 26835 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:17.548753 25986 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111) [suppressed 9 similar messages]
02:49:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:17.550125 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:17.550315 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.550455 25986 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.550760 26835 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:49:17.627 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:17.628 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:17.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.628 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:17.768 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:17.768 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:17.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:17.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.770 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:17.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:17.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.885298 26835 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:49:17.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.885506 26835 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:17.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.886221 26835 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:17.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:17.887687 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:17.887871 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.888032 25986 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:17.888340 26835 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:49:18.007 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:18.008 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:18.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:18.008 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:18.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:18.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:18.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:18.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:18.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:18.048 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:18.048 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:18.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:18.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:18.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:18.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:18.050 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:18.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:18.051 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:18.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:18.410246 26842 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:49:18.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:18.410434 26842 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:18.411087 26842 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:18.413431 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:18.413661 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:18.413784 25986 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:18.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:18.414039 26842 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:49:18.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:18.833746 26842 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:49:18.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:18.833966 26842 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:18.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:18.834722 26842 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:18.836133 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:18.836369 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:18.836483 25986 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:18.836760 26842 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:49:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:19.191210 26842 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:49:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:19.191421 26842 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:19.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:19.192087 26842 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:19.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:19.193696 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:19.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:19.193874 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:19.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:19.194002 25986 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:19.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:19.194329 26842 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:49:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:19.592094 26842 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:49:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:19.592301 26842 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:19.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:19.592947 26842 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:19.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:19.594466 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:19.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:19.594713 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:19.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:19.594848 25986 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:19.595121 26842 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:49:19.748 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:19.748 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:19.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:19.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:19.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:19.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:19.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:19.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:19.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:22.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:22.123139 25970 master_runner.cc:186] Error getting master registration for 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111),
02:49:22.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:22.123370 25970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:49:22.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:22.329885 26870 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:49:22.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:22.330124 26870 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:22.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:22.330916 26870 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:22.333001 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:22.333288 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:22.333444 25986 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:22.333745 26870 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:49:23.747 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:23.748 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:23.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:23.748 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:23.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:23.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:23.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:23.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:23.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:25.548 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:25.549 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:25.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:25.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:25.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:25.550 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:25.550 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:25.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:25.551 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:27.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:27.126348 25970 master_runner.cc:186] Error getting master registration for 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111),
02:49:27.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:27.126554 25970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:49:27.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:27.388913 26904 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:49:27.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:27.389243 26904 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:27.389966 26904 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:27.390910 25986 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111) [suppressed 13 similar messages]
02:49:27.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:27.392248 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:27.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:27.392387 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:27.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:27.392517 25985 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:27.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:27.392776 26904 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:49:28.927 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:28.928 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:28.928 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:28.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:28.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:28.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:28.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:28.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:28.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:29.547 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:29.548 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:29.548 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:29.548 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:29.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:29.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:29.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:29.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:29.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:31.867 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:31.868 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:31.868 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:31.868 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:31.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:31.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:31.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:31.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:31.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:32.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:32.129824 25970 master_runner.cc:186] Error getting master registration for 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111),
02:49:32.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:32.130009 25970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:49:33.308 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:33.308 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:33.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:33.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:33.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:33.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:33.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:33.310 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:33.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:33.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:33.962877 26950 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:49:33.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:33.963107 26950 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:33.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:33.963846 26950 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:33.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:33.964939 25985 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:49:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:33.966446 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:33.966651 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:33.966811 25986 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:33.967154 26950 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:49:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:34.534276 26960 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:49:34.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:34.534476 26960 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:34.535413 26960 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:34.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:34.537616 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:34.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:34.537849 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:34.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:34.537947 25986 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:34.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:34.538218 26960 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:49:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:37.133170 25970 master_runner.cc:186] Error getting master registration for 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111),
02:49:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:37.133431 25970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:49:37.368 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:37.368 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:37.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:37.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:37.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:37.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:37.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:37.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:37.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:40.768 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:40.768 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:40.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:40.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:40.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:40.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:40.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:40.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:40.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:42.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:42.136590 25970 master_runner.cc:186] Error getting master registration for 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111),
02:49:42.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:42.136900 25970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:49:44.347 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:44.348 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:44.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:44.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:44.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:44.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:44.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:44.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:44.351 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:46.167 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:46.168 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:46.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:46.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:46.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:46.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:46.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:46.170 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:46.171 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:47.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:47.139923 25970 master_runner.cc:186] Error getting master registration for 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111),
02:49:47.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:47.140133 25970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:49:47.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:47.717311 27047 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:49:47.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:47.717504 27047 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:49:47.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:47.718200 27047 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:49:47.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:47.719520 25986 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111) [suppressed 3 similar messages]
02:49:47.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:47.720986 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:49:47.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:49:47.721233 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:49:47.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:47.721375 25985 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:49:47.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:47.721655 27047 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:49:48.928 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:48.929 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:48.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:48.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:48.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:48.930 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:48.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:48.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:48.931 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:51.587 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:51.588 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:51.588 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:51.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:51.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:51.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:51.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:51.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:51.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:52.143163 25970 master_runner.cc:186] Error getting master registration for 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111),
02:49:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:52.143354 25970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:49:53.227 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:53.228 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:53.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:53.228 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:53.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:53.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:53.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:53.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:53.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:54.228 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:54.229 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:54.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:54.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:54.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:54.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:54.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:54.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:54.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:54.768 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:54.768 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:54.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:54.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:54.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:54.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:54.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:54.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:54.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:57.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:57.146799 25970 master_runner.cc:186] Error getting master registration for 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111),
02:49:57.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:49:57.146984 25970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:49:57.908 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:57.908 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:57.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:57.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:57.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:57.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:57.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:57.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:57.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:49:59.007 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:49:59.008 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:49:59.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:59.008 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:59.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:49:59.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:59.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:59.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:49:59.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:50:02.128 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:50:02.129 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:50:02.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:50:02.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:50:02.130 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:50:02.130 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:50:02.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:50:02.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:50:02.131 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:50:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:02.150246 25970 master_runner.cc:186] Error getting master registration for 127.25.89.253:46647: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111),
02:50:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:02.150444 25970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:50:03.947 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:50:03.948 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:50:03.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:50:03.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:50:03.948 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:50:03.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:50:03.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:50:03.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:50:03.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:50:04.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:04.398811 27154 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9c0f8ffb53444dc5bddc487da2466d59)
02:50:04.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:04.399030 27154 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3af63bec99064cbbb230e2f985fd1a34" member_type: VOTER last_known_addr { host: "127.25.89.254" port: 40383 } } peers { permanent_uuid: "9c0f8ffb53444dc5bddc487da2466d59" member_type: VOTER last_known_addr { host: "127.25.89.253" port: 46647 } } peers { permanent_uuid: "57d79a0a4f074da4b31478e6fdaa1954" member_type: VOTER last_known_addr { host: "127.25.89.252" port: 40643 } }
02:50:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:04.400035 27154 leader_election.cc:288] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647), 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643)
02:50:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:50:04.401180 25985 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:50:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:50:04.402486 25985 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9c0f8ffb53444dc5bddc487da2466d59 (127.25.89.253:46647): Network error: Client connection negotiation failed: client connection to 127.25.89.253:46647: connect: Connection refused (error 111)
02:50:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:50:04.402661 25986 leader_election.cc:334] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 57d79a0a4f074da4b31478e6fdaa1954 (127.25.89.252:40643): Network error: Client connection negotiation failed: client connection to 127.25.89.252:40643: connect: Connection refused (error 111)
02:50:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:04.402801 25986 leader_election.cc:302] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3af63bec99064cbbb230e2f985fd1a34; no voters: 57d79a0a4f074da4b31478e6fdaa1954, 9c0f8ffb53444dc5bddc487da2466d59
02:50:04.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:04.403095 27154 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3af63bec99064cbbb230e2f985fd1a34 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
02:50:06.328 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.253:46647(127.25.89.253:46647)] connecting to peer
02:50:06.328 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.25.89.252:40643(127.25.89.252:40643)] connecting to peer
02:50:06.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:50:06.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.89.253:46647(127.25.89.253:46647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:50:06.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:50:06.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.253:46647: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647
02:50:06.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.252:40643(127.25.89.252:40643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:50:06.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.89.252:40643: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
02:50:06.331 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383; will retry
02:50:06.332 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=32, TimeoutTracker(timeout=50000, elapsed=48977), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [2ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [2ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [3ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [14ms] refreshing cache from master, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [16ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [16ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [17ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [17ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [34ms] refreshing cache from master, [34ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [36ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [36ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [37ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [37ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [54ms] refreshing cache from master, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [56ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [56ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [57ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [58ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [75ms] refreshing cache from master, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [76ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [76ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [77ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [77ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [78ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [114ms] refreshing cache from master, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [116ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [116ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [117ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [118ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [174ms] refreshing cache from master, [174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [176ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [176ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [177ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [177ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [274ms] refreshing cache from master, [274ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [275ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [276ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [277ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [277ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [414ms] refreshing cache from master, [414ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [416ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [416ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [417ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [418ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [656ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [656ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [657ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [657ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [694ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [696ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [697ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [697ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [698ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [2394ms] refreshing cache from master, [2395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [2395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [2396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [2396ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [2396ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [2397ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [2398ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [6394ms] refreshing cache from master, [6394ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [6395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [6395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [6396ms] too many traces: truncated at 100 traces, deferred=Deferred@1186310143(state=PENDING, result=null, callback=(continuation of Deferred@218451305 after retry RPC after error@71872019) -> (continuation of Deferred@2146249290 after retry RPC after error@344780823) -> (continuation of Deferred@860195306 after retry RPC after error@1916567975) -> (continuation of Deferred@1687505616 after retry RPC after error@392823325) -> (continuation of Deferred@142590915 after retry RPC after error@517495506) -> (continuation of Deferred@1356156947 after retry RPC after error@392747774) -> (continuation of Deferred@1635738123 after retry RPC after error@1034791484) -> (continuation of Deferred@1199710290 after retry RPC after error@376221046) -> (continuation of Deferred@211413642 after retry RPC after error@1555389807) -> (continuation of Deferred@182679067 after retry RPC after error@2031578412) -> (continuation of Deferred@974718270 after retry RPC after error@1278310711) -> (continuation of Deferred@1567853828 after retry RPC after error@525333899) -> (continuation of Deferred@1837970345 after retry RPC after error@716314606) -> (continuation of Deferred@1802019857 after retry RPC after error@200013658) -> (continuation of Deferred@260017653 after retry RPC after error@902358675) -> (continuation of Deferred@456037399 after retry RPC after error@1336664335) -> (continuation of Deferred@1011404074 after retry RPC after error@1313228878) -> (continuation of Deferred@564655037 after retry RPC after error@831839519) -> (continuation of Deferred@1271883674 after retry RPC after error@926372501) -> (continuation of Deferred@697525717 after retry RPC after error@1048066525) -> (continuation of Deferred@247163455 after retry RPC after error@51716743) -> (continuation of Deferred@823804669 after retry RPC after error@1322460114) -> (continuation of Deferred@460083877 after retry RPC after error@826755669) -> (continuation of Deferred@1386867498 after retry RPC after error@95653662) -> (continuation of Deferred@2083720311 after retry RPC after error@1999500358) -> (continuation of Deferred@1782624562 after retry RPC after error@1930565659) -> (continuation of Deferred@396817082 after retry RPC after error@254913484) -> (continuation of Deferred@1612141113 after retry RPC after error@52841696) -> (continuation of Deferred@599084316 after retry RPC after error@567864685) -> (continuation of Deferred@2072311285 after retry RPC after error@1249175337) -> (continuation of Deferred@1126211286 after retry RPC after error@628469850), errback=(continuation of Deferred@218451305 after retry RPC after error@71872019) -> (continuation of Deferred@2146249290 after retry RPC after error@344780823) -> (continuation of Deferred@860195306 after retry RPC after error@1916567975) -> (continuation of Deferred@1687505616 after retry RPC after error@392823325) -> (continuation of Deferred@142590915 after retry RPC after error@517495506) -> (continuation of Deferred@1356156947 after retry RPC after error@392747774) -> (continuation of Deferred@1635738123 after retry RPC after error@1034791484) -> (continuation of Deferred@1199710290 after retry RPC after error@376221046) -> (continuation of Deferred@211413642 after retry RPC after error@1555389807) -> (continuation of Deferred@182679067 after retry RPC after error@2031578412) -> (continuation of Deferred@974718270 after retry RPC after error@1278310711) -> (continuation of Deferred@1567853828 after retry RPC after error@525333899) -> (continuation of Deferred@1837970345 after retry RPC after error@716314606) -> (continuation of Deferred@1802019857 after retry RPC after error@200013658) -> (continuation of Deferred@260017653 after retry RPC after error@902358675) -> (continuation of Deferred@456037399 after retry RPC after error@1336664335) -> (continuation of Deferred@1011404074 after retry RPC after error@1313228878) -> (continuation of Deferred@564655037 after retry RPC after error@831839519) -> (continuation of Deferred@1271883674 after retry RPC after error@926372501) -> (continuation of Deferred@697525717 after retry RPC after error@1048066525) -> (continuation of Deferred@247163455 after retry RPC after error@51716743) -> (continuation of Deferred@823804669 after retry RPC after error@1322460114) -> (continuation of Deferred@460083877 after retry RPC after error@826755669) -> (continuation of Deferred@1386867498 after retry RPC after error@95653662) -> (continuation of Deferred@2083720311 after retry RPC after error@1999500358) -> (continuation of Deferred@1782624562 after retry RPC after error@1930565659) -> (continuation of Deferred@396817082 after retry RPC after error@254913484) -> (continuation of Deferred@1612141113 after retry RPC after error@52841696) -> (continuation of Deferred@599084316 after retry RPC after error@567864685) -> (continuation of Deferred@2072311285 after retry RPC after error@1249175337) -> (continuation of Deferred@1126211286 after retry RPC after error@628469850)))
02:50:06.334 [DEBUG - main] (Connection.java:575) [peer master-127.25.89.254:40383(127.25.89.254:40383)] disconnecting while in state READY
02:50:06.334 [DEBUG - main] (Connection.java:575) [peer master-127.25.89.252:40643(127.25.89.252:40643)] disconnecting while in state TERMINATED
02:50:06.334 [DEBUG - main] (Connection.java:575) [peer 60171f79b6734d19a4b20f9b7f5ce54a(127.25.89.193:35885)] disconnecting while in state READY
02:50:06.334 [DEBUG - main] (Connection.java:575) [peer master-127.25.89.253:46647(127.25.89.253:46647)] disconnecting while in state TERMINATED
02:50:06.335 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 60171f79b6734d19a4b20f9b7f5ce54a(127.25.89.193:35885)] handling channelInactive
02:50:06.335 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 60171f79b6734d19a4b20f9b7f5ce54a(127.25.89.193:35885)] cleaning up while in state READY due to: connection closed
02:50:06.335 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:50:06.336 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.25.89.254:40383(127.25.89.254:40383)] handling channelInactive
02:50:06.336 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.89.254:40383(127.25.89.254:40383)] cleaning up while in state READY due to: connection closed
02:50:06.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.336730 25959 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu pid 26252
02:50:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.346033 26403 generic_service.cc:196] Checking for leaks (request via RPC)
02:50:06.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.453967 25959 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu with pid 26252
02:50:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:50:06.470861 26572 connection.cc:535] server connection from 127.25.89.193:35763 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:50:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.471230 25959 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu pid 26413
02:50:06.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.479573 26544 generic_service.cc:196] Checking for leaks (request via RPC)
02:50:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.606550 25959 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu with pid 26413
02:50:06.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 02:50:06.626420 26570 connection.cc:535] server connection from 127.25.89.194:39937 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:50:06.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.626950 25959 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu pid 26554
02:50:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.634462 26686 generic_service.cc:196] Checking for leaks (request via RPC)
02:50:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.739645 25959 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu with pid 26554
02:50:06.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.755868 25959 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu pid 25970
02:50:06.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.757272 26041 generic_service.cc:196] Checking for leaks (request via RPC)
02:50:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 02:50:06.874972 25959 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQjj_5e/build/asan/bin/kudu with pid 25970
02:50:06.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T02:50:06Z chronyd exiting
02:50:07.024 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=32, TimeoutTracker(timeout=50000, elapsed=48977), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [2ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [2ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [3ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [14ms] refreshing cache from master, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [16ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [16ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [17ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [17ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [34ms] refreshing cache from master, [34ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [36ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [36ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [37ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [37ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [54ms] refreshing cache from master, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [56ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [56ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [57ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [58ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [75ms] refreshing cache from master, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [76ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [76ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [77ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [77ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [78ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [114ms] refreshing cache from master, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [116ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [116ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [117ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [118ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [174ms] refreshing cache from master, [174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [176ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [176ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [177ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [177ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [274ms] refreshing cache from master, [274ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [275ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [276ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [277ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [277ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [414ms] refreshing cache from master, [414ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [416ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [416ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [417ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [418ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [656ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [656ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [657ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [657ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [694ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [696ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [697ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [697ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [698ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [2394ms] refreshing cache from master, [2395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [2395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [2396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [2396ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [2396ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [2397ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [2398ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [6394ms] refreshing cache from master, [6394ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [6395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [6395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [6396ms] too many traces: truncated at 100 traces, deferred=Deferred@1186310143(state=PENDING, result=null, callback=(continuation of Deferred@218451305 after retry RPC after error@71872019) -> (continuation of Deferred@2146249290 after retry RPC after error@344780823) -> (continuation of Deferred@860195306 after retry RPC after error@1916567975) -> (continuation of Deferred@1687505616 after retry RPC after error@392823325) -> (continuation of Deferred@142590915 after retry RPC after error@517495506) -> (continuation of Deferred@1356156947 after retry RPC after error@392747774) -> (continuation of Deferred@1635738123 after retry RPC after error@1034791484) -> (continuation of Deferred@1199710290 after retry RPC after error@376221046) -> (continuation of Deferred@211413642 after retry RPC after error@1555389807) -> (continuation of Deferred@182679067 after retry RPC after error@2031578412) -> (continuation of Deferred@974718270 after retry RPC after error@1278310711) -> (continuation of Deferred@1567853828 after retry RPC after error@525333899) -> (continuation of Deferred@1837970345 after retry RPC after error@716314606) -> (continuation of Deferred@1802019857 after retry RPC after error@200013658) -> (continuation of Deferred@260017653 after retry RPC after error@902358675) -> (continuation of Deferred@456037399 after retry RPC after error@1336664335) -> (continuation of Deferred@1011404074 after retry RPC after error@1313228878) -> (continuation of Deferred@564655037 after retry RPC after error@831839519) -> (continuation of Deferred@1271883674 after retry RPC after error@926372501) -> (continuation of Deferred@697525717 after retry RPC after error@1048066525) -> (continuation of Deferred@247163455 after retry RPC after error@51716743) -> (continuation of Deferred@823804669 after retry RPC after error@1322460114) -> (continuation of Deferred@460083877 after retry RPC after error@826755669) -> (continuation of Deferred@1386867498 after retry RPC after error@95653662) -> (continuation of Deferred@2083720311 after retry RPC after error@1999500358) -> (continuation of Deferred@1782624562 after retry RPC after error@1930565659) -> (continuation of Deferred@396817082 after retry RPC after error@254913484) -> (continuation of Deferred@1612141113 after retry RPC after error@52841696) -> (continuation of Deferred@599084316 after retry RPC after error@567864685) -> (continuation of Deferred@2072311285 after retry RPC after error@1249175337) -> (continuation of Deferred@1126211286 after retry RPC after error@628469850), errback=(continuation of Deferred@218451305 after retry RPC after error@71872019) -> (continuation of Deferred@2146249290 after retry RPC after error@344780823) -> (continuation of Deferred@860195306 after retry RPC after error@1916567975) -> (continuation of Deferred@1687505616 after retry RPC after error@392823325) -> (continuation of Deferred@142590915 after retry RPC after error@517495506) -> (continuation of Deferred@1356156947 after retry RPC after error@392747774) -> (continuation of Deferred@1635738123 after retry RPC after error@1034791484) -> (continuation of Deferred@1199710290 after retry RPC after error@376221046) -> (continuation of Deferred@211413642 after retry RPC after error@1555389807) -> (continuation of Deferred@182679067 after retry RPC after error@2031578412) -> (continuation of Deferred@974718270 after retry RPC after error@1278310711) -> (continuation of Deferred@1567853828 after retry RPC after error@525333899) -> (continuation of Deferred@1837970345 after retry RPC after error@716314606) -> (continuation of Deferred@1802019857 after retry RPC after error@200013658) -> (continuation of Deferred@260017653 after retry RPC after error@902358675) -> (continuation of Deferred@456037399 after retry RPC after error@1336664335) -> (continuation of Deferred@1011404074 after retry RPC after error@1313228878) -> (continuation of Deferred@564655037 after retry RPC after error@831839519) -> (continuation of Deferred@1271883674 after retry RPC after error@926372501) -> (continuation of Deferred@697525717 after retry RPC after error@1048066525) -> (continuation of Deferred@247163455 after retry RPC after error@51716743) -> (continuation of Deferred@823804669 after retry RPC after error@1322460114) -> (continuation of Deferred@460083877 after retry RPC after error@826755669) -> (continuation of Deferred@1386867498 after retry RPC after error@95653662) -> (continuation of Deferred@2083720311 after retry RPC after error@1999500358) -> (continuation of Deferred@1782624562 after retry RPC after error@1930565659) -> (continuation of Deferred@396817082 after retry RPC after error@254913484) -> (continuation of Deferred@1612141113 after retry RPC after error@52841696) -> (continuation of Deferred@599084316 after retry RPC after error@567864685) -> (continuation of Deferred@2072311285 after retry RPC after error@1249175337) -> (continuation of Deferred@1126211286 after retry RPC after error@628469850)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.89.252:40643
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=CommitTransaction, tablet=null, attempt=32, TimeoutTracker(timeout=50000, elapsed=48977), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [2ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [2ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [3ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [14ms] refreshing cache from master, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [15ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [16ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [16ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [17ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [17ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [34ms] refreshing cache from master, [34ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [36ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [36ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [37ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [37ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [54ms] refreshing cache from master, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [55ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [56ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [56ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [57ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [58ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [75ms] refreshing cache from master, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [76ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [76ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [77ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [77ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [78ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [114ms] refreshing cache from master, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [116ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [116ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [117ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [118ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [174ms] refreshing cache from master, [174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [176ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [176ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [177ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [177ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [274ms] refreshing cache from master, [274ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [275ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [275ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [276ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [277ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [277ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [414ms] refreshing cache from master, [414ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [416ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [416ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [417ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [418ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [656ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [656ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [657ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [657ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [694ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [696ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [697ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [697ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [698ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [2394ms] refreshing cache from master, [2395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [2395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [2396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [2396ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.253:46647: Network error: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647, [2396ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.252:40643: Network error: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [2397ms] Sub RPC ConnectToMaster: received response from server master-127.25.89.254:40383: OK, [2398ms] delaying RPC due to: Service unavailable: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643, [6394ms] refreshing cache from master, [6394ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.253:46647, [6395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.252:40643, [6395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.89.254:40383, [6396ms] too many traces: truncated at 100 traces, deferred=Deferred@1186310143(state=PENDING, result=null, callback=(continuation of Deferred@218451305 after retry RPC after error@71872019) -> (continuation of Deferred@2146249290 after retry RPC after error@344780823) -> (continuation of Deferred@860195306 after retry RPC after error@1916567975) -> (continuation of Deferred@1687505616 after retry RPC after error@392823325) -> (continuation of Deferred@142590915 after retry RPC after error@517495506) -> (continuation of Deferred@1356156947 after retry RPC after error@392747774) -> (continuation of Deferred@1635738123 after retry RPC after error@1034791484) -> (continuation of Deferred@1199710290 after retry RPC after error@376221046) -> (continuation of Deferred@211413642 after retry RPC after error@1555389807) -> (continuation of Deferred@182679067 after retry RPC after error@2031578412) -> (continuation of Deferred@974718270 after retry RPC after error@1278310711) -> (continuation of Deferred@1567853828 after retry RPC after error@525333899) -> (continuation of Deferred@1837970345 after retry RPC after error@716314606) -> (continuation of Deferred@1802019857 after retry RPC after error@200013658) -> (continuation of Deferred@260017653 after retry RPC after error@902358675) -> (continuation of Deferred@456037399 after retry RPC after error@1336664335) -> (continuation of Deferred@1011404074 after retry RPC after error@1313228878) -> (continuation of Deferred@564655037 after retry RPC after error@831839519) -> (continuation of Deferred@1271883674 after retry RPC after error@926372501) -> (continuation of Deferred@697525717 after retry RPC after error@1048066525) -> (continuation of Deferred@247163455 after retry RPC after error@51716743) -> (continuation of Deferred@823804669 after retry RPC after error@1322460114) -> (continuation of Deferred@460083877 after retry RPC after error@826755669) -> (continuation of Deferred@1386867498 after retry RPC after error@95653662) -> (continuation of Deferred@2083720311 after retry RPC after error@1999500358) -> (continuation of Deferred@1782624562 after retry RPC after error@1930565659) -> (continuation of Deferred@396817082 after retry RPC after error@254913484) -> (continuation of Deferred@1612141113 after retry RPC after error@52841696) -> (continuation of Deferred@599084316 after retry RPC after error@567864685) -> (continuation of Deferred@2072311285 after retry RPC after error@1249175337) -> (continuation of Deferred@1126211286 after retry RPC after error@628469850), errback=(continuation of Deferred@218451305 after retry RPC after error@71872019) -> (continuation of Deferred@2146249290 after retry RPC after error@344780823) -> (continuation of Deferred@860195306 after retry RPC after error@1916567975) -> (continuation of Deferred@1687505616 after retry RPC after error@392823325) -> (continuation of Deferred@142590915 after retry RPC after error@517495506) -> (continuation of Deferred@1356156947 after retry RPC after error@392747774) -> (continuation of Deferred@1635738123 after retry RPC after error@1034791484) -> (continuation of Deferred@1199710290 after retry RPC after error@376221046) -> (continuation of Deferred@211413642 after retry RPC after error@1555389807) -> (continuation of Deferred@182679067 after retry RPC after error@2031578412) -> (continuation of Deferred@974718270 after retry RPC after error@1278310711) -> (continuation of Deferred@1567853828 after retry RPC after error@525333899) -> (continuation of Deferred@1837970345 after retry RPC after error@716314606) -> (continuation of Deferred@1802019857 after retry RPC after error@200013658) -> (continuation of Deferred@260017653 after retry RPC after error@902358675) -> (continuation of Deferred@456037399 after retry RPC after error@1336664335) -> (continuation of Deferred@1011404074 after retry RPC after error@1313228878) -> (continuation of Deferred@564655037 after retry RPC after error@831839519) -> (continuation of Deferred@1271883674 after retry RPC after error@926372501) -> (continuation of Deferred@697525717 after retry RPC after error@1048066525) -> (continuation of Deferred@247163455 after retry RPC after error@51716743) -> (continuation of Deferred@823804669 after retry RPC after error@1322460114) -> (continuation of Deferred@460083877 after retry RPC after error@826755669) -> (continuation of Deferred@1386867498 after retry RPC after error@95653662) -> (continuation of Deferred@2083720311 after retry RPC after error@1999500358) -> (continuation of Deferred@1782624562 after retry RPC after error@1930565659) -> (continuation of Deferred@396817082 after retry RPC after error@254913484) -> (continuation of Deferred@1612141113 after retry RPC after error@52841696) -> (continuation of Deferred@599084316 after retry RPC after error@567864685) -> (continuation of Deferred@2072311285 after retry RPC after error@1249175337) -> (continuation of Deferred@1126211286 after retry RPC after error@628469850)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.89.253:46647,127.25.89.252:40643,127.25.89.254:40383) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.253:46647(127.25.89.253:46647): Connection refused: /127.25.89.253:46647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.89.252:40643(127.25.89.252:40643): Connection refused: /127.25.89.252:40643
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.89.252:40643
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)