03:00:15.314 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:00:15.314 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task7KZUIc/test-tmp
03:00:15.314 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
03:00:15.314 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task7KZUIc/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
03:00:15.316 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:00:15.646 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:00:15.646 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:00:15.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:15.649125 4045 mini_oidc.cc:150] Starting JWKS server
03:00:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:15.651476 4045 webserver.cc:492] Webserver started at http://127.0.0.1:44967/ using document root <none> and password file <none>
03:00:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:15.651711 4045 mini_oidc.cc:186] Starting OIDC Discovery server
03:00:15.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:15.652169 4045 webserver.cc:492] Webserver started at http://127.0.0.1:42145/ using document root <none> and password file <none>
03:00:15.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-10T03:00:15Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:00:15.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-10T03:00:15Z Disabled control of system clock
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:15.681484 4045 external_mini_cluster.cc:1366] Running /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/wal
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/data
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/logs
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/data/info.pb
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:00:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.126:33571
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.126
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44967/jwks.json
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:00:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:00:16.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.007076 4056 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:00:16.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.007395 4056 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:00:16.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.007475 4056 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:00:16.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.015733 4056 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:00:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.015851 4056 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:00:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.015898 4056 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:00:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.015937 4056 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:00:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.015991 4056 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:00:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.016028 4056 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:00:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.016077 4056 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44967/jwks.json
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.026091 4056 master_runner.cc:386] Master server non-default flags:
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/data
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/wal
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.126:33571
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44967/jwks.json
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/data/info.pb
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.126
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/logs
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 02:43:21 UTC on 24a791456cd2
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9440
03:00:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:00:16.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.027756 4056 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:00:16.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.029606 4056 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:00:16.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.037662 4062 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:16.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.037642 4061 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:16.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.037997 4064 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:16.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.038429 4056 server_base.cc:1047] running on GCE node
03:00:16.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.039222 4056 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:00:16.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.040621 4056 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:00:16.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.041847 4056 hybrid_clock.cc:648] HybridClock initialized: now 1765335616041785 us; error 57 us; skew 500 ppm
03:00:16.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.042204 4056 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:00:16.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.044382 4056 webserver.cc:492] Webserver started at http://127.3.243.126:46255/ using document root <none> and password file <none>
03:00:16.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.045156 4056 fs_manager.cc:362] Metadata directory not provided
03:00:16.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.045288 4056 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:00:16.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.045640 4056 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
03:00:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.048025 4056 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/data/instance:
03:00:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28b4d256f6804d7aab5e92586dc1b038"
03:00:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:16 on dist-test-slave-k7nm"
03:00:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.048815 4056 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/wal/instance:
03:00:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28b4d256f6804d7aab5e92586dc1b038"
03:00:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:16 on dist-test-slave-k7nm"
03:00:16.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.054188 4056 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.000s sys 0.004s
03:00:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.057592 4070 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.058933 4056 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.059120 4056 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/data,/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/wal
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28b4d256f6804d7aab5e92586dc1b038"
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:16 on dist-test-slave-k7nm"
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.059331 4056 fs_report.cc:389] FS layout report
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/wal
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/wal
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/data/data
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:00:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:00:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.077771 4056 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:00:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.078724 4056 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:00:16.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.079020 4056 kserver.cc:163] Server-wide thread pool size limit: 3276
03:00:16.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.102097 4056 rpc_server.cc:307] RPC server started. Bound to: 127.3.243.126:33571
03:00:16.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.102188 4131 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.243.126:33571 every 8 connection(s)
03:00:16.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.103730 4056 server_base.cc:1179] Dumped server information to /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/data/info.pb
03:00:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.108157 4132 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:16.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.109331 4045 external_mini_cluster.cc:1428] Started /tmp/dist-test-task7KZUIc/build/asan/bin/kudu as pid 4056
03:00:16.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.109619 4045 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-0/wal/instance
03:00:16.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.111821 4045 external_mini_cluster.cc:1366] Running /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/wal
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/data
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/logs
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/data/info.pb
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.125:39339
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.125
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:00:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:16.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:00:16.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44967/jwks.json
03:00:16.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:00:16.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:00:16.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.114331 4132 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } has no permanent_uuid. Determining permanent_uuid...
03:00:16.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.130798 4132 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } has no permanent_uuid. Determining permanent_uuid...
03:00:16.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.132136 4071 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.243.125:39339: connect: Connection refused (error 111)
03:00:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.133844 4132 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.125:39339: Network error: Client connection negotiation failed: client connection to 127.3.243.125:39339: connect: Connection refused (error 111)
03:00:16.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.183226 4132 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } attempt: 1
03:00:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.185199 4132 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.125:39339: Network error: Client connection negotiation failed: client connection to 127.3.243.125:39339: connect: Connection refused (error 111)
03:00:16.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.253437 4132 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } attempt: 2
03:00:16.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.255333 4132 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.125:39339: Network error: Client connection negotiation failed: client connection to 127.3.243.125:39339: connect: Connection refused (error 111)
03:00:16.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.346578 4132 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } attempt: 3
03:00:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.348421 4132 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.125:39339: Network error: Client connection negotiation failed: client connection to 127.3.243.125:39339: connect: Connection refused (error 111)
03:00:16.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.449113 4134 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:00:16.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.449512 4134 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:00:16.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.449628 4134 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:00:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.457875 4134 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:00:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.457985 4134 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:00:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.458056 4134 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:00:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.458118 4134 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:00:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.458191 4134 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:00:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.458247 4134 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:00:16.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.458308 4134 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44967/jwks.json
03:00:16.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.468483 4134 master_runner.cc:386] Master server non-default flags:
03:00:16.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:16.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:16.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:16.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/data
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/wal
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.125:39339
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44967/jwks.json
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/data/info.pb
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.125
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/logs
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:00:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
03:00:16.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:00:16.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 02:43:21 UTC on 24a791456cd2
03:00:16.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9440
03:00:16.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:00:16.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.470566 4134 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:00:16.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.472455 4134 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:00:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.480235 4143 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.480262 4142 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:16.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.480968 4145 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:16.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.481488 4134 server_base.cc:1047] running on GCE node
03:00:16.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.482256 4134 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:00:16.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.483630 4134 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:00:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.484877 4134 hybrid_clock.cc:648] HybridClock initialized: now 1765335616484803 us; error 38 us; skew 500 ppm
03:00:16.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.485224 4134 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:00:16.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.487322 4134 webserver.cc:492] Webserver started at http://127.3.243.125:35657/ using document root <none> and password file <none>
03:00:16.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.488018 4134 fs_manager.cc:362] Metadata directory not provided
03:00:16.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.488147 4134 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:00:16.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.488490 4134 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
03:00:16.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.490928 4134 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/data/instance:
03:00:16.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b98d89b2a2444f97bab4ad389ef0c173"
03:00:16.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:16 on dist-test-slave-k7nm"
03:00:16.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.491662 4134 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/wal/instance:
03:00:16.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b98d89b2a2444f97bab4ad389ef0c173"
03:00:16.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:16 on dist-test-slave-k7nm"
03:00:16.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.491639 4132 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } attempt: 4
03:00:16.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.493400 4132 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.125:39339: Network error: Client connection negotiation failed: client connection to 127.3.243.125:39339: connect: Connection refused (error 111)
03:00:16.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.496941 4134 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.001s sys 0.006s
03:00:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.500252 4151 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:00:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.501629 4134 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.501801 4134 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/data,/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/wal
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b98d89b2a2444f97bab4ad389ef0c173"
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:16 on dist-test-slave-k7nm"
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.501982 4134 fs_report.cc:389] FS layout report
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/wal
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/wal
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/data/data
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:00:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:00:16.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.525364 4134 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:00:16.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.526338 4134 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:00:16.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.526649 4134 kserver.cc:163] Server-wide thread pool size limit: 3276
03:00:16.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.548610 4134 rpc_server.cc:307] RPC server started. Bound to: 127.3.243.125:39339
03:00:16.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.548642 4212 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.243.125:39339 every 8 connection(s)
03:00:16.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.550194 4134 server_base.cc:1179] Dumped server information to /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/data/info.pb
03:00:16.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.551317 4045 external_mini_cluster.cc:1428] Started /tmp/dist-test-task7KZUIc/build/asan/bin/kudu as pid 4134
03:00:16.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.551492 4045 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-1/wal/instance
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.552436 4045 external_mini_cluster.cc:1366] Running /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/wal
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/data
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/logs
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/data/info.pb
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:00:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.124:41589
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.124
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44967/jwks.json
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:00:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:00:16.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.554643 4213 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:16.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.561939 4213 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } has no permanent_uuid. Determining permanent_uuid...
03:00:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.579116 4213 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } has no permanent_uuid. Determining permanent_uuid...
03:00:16.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.589882 4213 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } has no permanent_uuid. Determining permanent_uuid...
03:00:16.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.591020 4154 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:16.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.592586 4213 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.124:41589: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:16.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.641889 4213 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } attempt: 1
03:00:16.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.643837 4213 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.124:41589: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.712077 4213 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } attempt: 2
03:00:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.714084 4213 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.124:41589: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:16.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.792698 4132 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } attempt: 5
03:00:16.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.799023 4132 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } has no permanent_uuid. Determining permanent_uuid...
03:00:16.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.800704 4132 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.124:41589: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:16.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.805317 4213 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } attempt: 3
03:00:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.807117 4213 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.124:41589: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.852003 4132 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } attempt: 1
03:00:16.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.853945 4132 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.124:41589: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.888592 4215 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:00:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.888953 4215 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:00:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.889034 4215 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:00:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.897154 4215 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:00:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.897256 4215 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:00:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.897302 4215 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:00:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.897341 4215 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:00:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.897399 4215 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:00:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.897439 4215 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:00:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.897481 4215 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44967/jwks.json
03:00:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.909602 4215 master_runner.cc:386] Master server non-default flags:
03:00:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:00:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/data
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/wal
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.124:41589
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44967/jwks.json
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/data/info.pb
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.124
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/logs
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 02:43:21 UTC on 24a791456cd2
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9440
03:00:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:00:16.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.911240 4215 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:00:16.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.913055 4215 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:00:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.920842 4224 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.921027 4223 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.921459 4226 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:16.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.921940 4215 server_base.cc:1047] running on GCE node
03:00:16.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.922197 4132 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } attempt: 2
03:00:16.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.922842 4215 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:00:16.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.923890 4132 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.124:41589: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:16.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.924360 4215 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:00:16.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.925566 4215 hybrid_clock.cc:648] HybridClock initialized: now 1765335616925507 us; error 38 us; skew 500 ppm
03:00:16.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.925861 4215 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:00:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.927983 4215 webserver.cc:492] Webserver started at http://127.3.243.124:36959/ using document root <none> and password file <none>
03:00:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.928666 4215 fs_manager.cc:362] Metadata directory not provided
03:00:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.928768 4215 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:00:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.929126 4215 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
03:00:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.931512 4215 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/data/instance:
03:00:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fdd32a3c604d4763b45610d82e3138c4"
03:00:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:16 on dist-test-slave-k7nm"
03:00:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.932255 4215 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/wal/instance:
03:00:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fdd32a3c604d4763b45610d82e3138c4"
03:00:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:16 on dist-test-slave-k7nm"
03:00:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.937474 4215 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
03:00:16.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.940757 4232 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.942183 4215 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.942342 4215 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/data,/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/wal
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fdd32a3c604d4763b45610d82e3138c4"
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:16 on dist-test-slave-k7nm"
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.942517 4215 fs_report.cc:389] FS layout report
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/wal
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/wal
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/data/data
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:00:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:00:16.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.950331 4213 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } attempt: 4
03:00:16.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:16.952174 4213 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.3.243.124:41589: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:16.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.979976 4215 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:00:16.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.980930 4215 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:00:16.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:16.981191 4215 kserver.cc:163] Server-wide thread pool size limit: 3276
03:00:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.003527 4215 rpc_server.cc:307] RPC server started. Bound to: 127.3.243.124:41589
03:00:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.003558 4293 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.243.124:41589 every 8 connection(s)
03:00:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.005172 4215 server_base.cc:1179] Dumped server information to /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/data/info.pb
03:00:17.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.009631 4294 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:17.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.011307 4045 external_mini_cluster.cc:1428] Started /tmp/dist-test-task7KZUIc/build/asan/bin/kudu as pid 4215
03:00:17.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.011523 4045 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/master-2/wal/instance
03:00:17.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.016955 4294 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } has no permanent_uuid. Determining permanent_uuid...
03:00:17.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.029898 4294 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } has no permanent_uuid. Determining permanent_uuid...
03:00:17.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.030093 4132 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } attempt: 3
03:00:17.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.038002 4294 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } has no permanent_uuid. Determining permanent_uuid...
03:00:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.046537 4132 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038: Bootstrap starting.
03:00:17.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.048337 4294 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4: Bootstrap starting.
03:00:17.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.050866 4132 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038: Neither blocks nor log segments found. Creating new log.
03:00:17.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.051587 4294 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4: Neither blocks nor log segments found. Creating new log.
03:00:17.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.051920 4132 log.cc:826] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038: Log is configured to *not* fsync() on all Append() calls
03:00:17.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.052592 4294 log.cc:826] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4: Log is configured to *not* fsync() on all Append() calls
03:00:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.055114 4132 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038: No bootstrap required, opened a new log
03:00:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.055310 4294 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4: No bootstrap required, opened a new log
03:00:17.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.060074 4132 raft_consensus.cc:359] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:17.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.060076 4294 raft_consensus.cc:359] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:17.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.060386 4132 raft_consensus.cc:385] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:17.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.060427 4294 raft_consensus.cc:385] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:17.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.060505 4132 raft_consensus.cc:740] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 28b4d256f6804d7aab5e92586dc1b038, State: Initialized, Role: FOLLOWER
03:00:17.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.060536 4294 raft_consensus.cc:740] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fdd32a3c604d4763b45610d82e3138c4, State: Initialized, Role: FOLLOWER
03:00:17.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.061236 4294 consensus_queue.cc:260] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.061932 4132 consensus_queue.cc:260] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.062867 4306 sys_catalog.cc:455] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.062906 4305 sys_catalog.cc:455] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.063203 4305 sys_catalog.cc:458] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [sys.catalog]: This master's current role is: FOLLOWER
03:00:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.063395 4306 sys_catalog.cc:458] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [sys.catalog]: This master's current role is: FOLLOWER
03:00:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.063823 4294 sys_catalog.cc:565] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [sys.catalog]: configured and running, proceeding with master startup.
03:00:17.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.063971 4132 sys_catalog.cc:565] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [sys.catalog]: configured and running, proceeding with master startup.
03:00:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.071676 4305 raft_consensus.cc:493] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:00:17.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.072664 4305 raft_consensus.cc:515] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:17.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.074297 4305 leader_election.cc:290] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 28b4d256f6804d7aab5e92586dc1b038 (127.3.243.126:33571), b98d89b2a2444f97bab4ad389ef0c173 (127.3.243.125:39339)
03:00:17.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.074661 4325 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:00:17.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.074764 4325 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:00:17.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.077674 4097 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fdd32a3c604d4763b45610d82e3138c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "28b4d256f6804d7aab5e92586dc1b038" is_pre_election: true
03:00:17.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.078240 4097 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fdd32a3c604d4763b45610d82e3138c4 in term 0.
03:00:17.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.080194 4328 catalog_manager.cc:1568] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:00:17.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.080322 4328 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:00:17.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.081328 4235 leader_election.cc:304] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 28b4d256f6804d7aab5e92586dc1b038, fdd32a3c604d4763b45610d82e3138c4; no voters:
03:00:17.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.081663 4076 tablet.cc:2378] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:00:17.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.081861 4305 raft_consensus.cc:2804] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:00:17.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.082026 4305 raft_consensus.cc:493] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:00:17.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.082154 4305 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [term 0 FOLLOWER]: Advancing to term 1
03:00:17.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.085539 4305 raft_consensus.cc:515] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:17.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.086421 4305 leader_election.cc:290] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [CANDIDATE]: Term 1 election: Requested vote from peers 28b4d256f6804d7aab5e92586dc1b038 (127.3.243.126:33571), b98d89b2a2444f97bab4ad389ef0c173 (127.3.243.125:39339)
03:00:17.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.086933 4097 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fdd32a3c604d4763b45610d82e3138c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "28b4d256f6804d7aab5e92586dc1b038"
03:00:17.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.087174 4097 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 0 FOLLOWER]: Advancing to term 1
03:00:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.090485 4097 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fdd32a3c604d4763b45610d82e3138c4 in term 1.
03:00:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.075095 4056 master_runner.cc:186] Error getting master registration for 127.3.243.125:39339: OK, instance_id { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" instance_seqno: 1765335616543615 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:00:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.093035 4056 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:00:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.093986 4235 leader_election.cc:304] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 28b4d256f6804d7aab5e92586dc1b038, fdd32a3c604d4763b45610d82e3138c4; no voters:
03:00:17.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.094867 4305 raft_consensus.cc:2804] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [term 1 FOLLOWER]: Leader election won for term 1
03:00:17.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.095875 4305 raft_consensus.cc:697] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [term 1 LEADER]: Becoming Leader. State: Replica: fdd32a3c604d4763b45610d82e3138c4, State: Running, Role: LEADER
03:00:17.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.096479 4305 consensus_queue.cc:237] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:17.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.101933 4331 sys_catalog.cc:455] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fdd32a3c604d4763b45610d82e3138c4. Latest consensus state: current_term: 1 leader_uuid: "fdd32a3c604d4763b45610d82e3138c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:17.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.102428 4331 sys_catalog.cc:458] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [sys.catalog]: This master's current role is: LEADER
03:00:17.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.094547 4215 master_runner.cc:186] Error getting master registration for 127.3.243.125:39339: OK, instance_id { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" instance_seqno: 1765335616543615 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:00:17.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.108012 4215 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:00:17.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.119772 4333 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:00:17.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.123294 4333 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:00:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.142323 4097 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 1 FOLLOWER]: Refusing update from remote peer fdd32a3c604d4763b45610d82e3138c4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:00:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.143621 4331 consensus_queue.cc:1048] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:00:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.164467 4306 sys_catalog.cc:455] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fdd32a3c604d4763b45610d82e3138c4. Latest consensus state: current_term: 1 leader_uuid: "fdd32a3c604d4763b45610d82e3138c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.164765 4306 sys_catalog.cc:458] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [sys.catalog]: This master's current role is: FOLLOWER
03:00:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.166105 4334 mvcc.cc:204] Tried to move back new op lower bound from 7230814687802867712 to 7230814687632736256. Current Snapshot: MvccSnapshot[applied={T|T < 7230814687802867712}]
03:00:17.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.168216 4331 sys_catalog.cc:455] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fdd32a3c604d4763b45610d82e3138c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:17.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.168480 4331 sys_catalog.cc:458] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [sys.catalog]: This master's current role is: LEADER
03:00:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.179476 4333 catalog_manager.cc:1357] Generated new cluster ID: cf189fddef194b5b91c14a3d072e7bc4
03:00:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.179610 4333 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:00:17.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.180402 4306 sys_catalog.cc:455] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fdd32a3c604d4763b45610d82e3138c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:17.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.180649 4306 sys_catalog.cc:458] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [sys.catalog]: This master's current role is: FOLLOWER
03:00:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.204502 4333 catalog_manager.cc:1380] Generated new certificate authority record
03:00:17.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.206347 4333 catalog_manager.cc:1514] Loading token signing keys...
03:00:17.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.243062 4333 catalog_manager.cc:6027] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4: Generated new TSK 0
03:00:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.244479 4333 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:00:17.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.251398 4213 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } attempt: 5
03:00:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.268694 4213 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173: Bootstrap starting.
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.270830 4045 external_mini_cluster.cc:1366] Running /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/wal
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/data
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/logs
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/data/info.pb
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.65:0
03:00:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.243.65
03:00:17.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.65
03:00:17.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:17.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:17.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:17.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:17.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:17.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:17.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:00:17.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:00:17.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:00:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.274770 4213 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173: Neither blocks nor log segments found. Creating new log.
03:00:17.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.276392 4213 log.cc:826] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173: Log is configured to *not* fsync() on all Append() calls
03:00:17.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.284130 4213 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173: No bootstrap required, opened a new log
03:00:17.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.292271 4213 raft_consensus.cc:359] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:17.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.292878 4213 raft_consensus.cc:385] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:17.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.293120 4213 raft_consensus.cc:740] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b98d89b2a2444f97bab4ad389ef0c173, State: Initialized, Role: FOLLOWER
03:00:17.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.294145 4213 consensus_queue.cc:260] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.301440 4341 sys_catalog.cc:455] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.303833 4213 sys_catalog.cc:565] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: configured and running, proceeding with master startup.
03:00:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.304195 4177 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fdd32a3c604d4763b45610d82e3138c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b98d89b2a2444f97bab4ad389ef0c173"
03:00:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.304625 4177 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 0 FOLLOWER]: Advancing to term 1
03:00:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.304000 4341 sys_catalog.cc:458] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: This master's current role is: FOLLOWER
03:00:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.307946 4177 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fdd32a3c604d4763b45610d82e3138c4 in term 1.
03:00:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.309610 4176 raft_consensus.cc:1275] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 1 FOLLOWER]: Refusing update from remote peer fdd32a3c604d4763b45610d82e3138c4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:00:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.303875 4178 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fdd32a3c604d4763b45610d82e3138c4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b98d89b2a2444f97bab4ad389ef0c173" is_pre_election: true
03:00:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.310750 4178 raft_consensus.cc:2376] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate fdd32a3c604d4763b45610d82e3138c4 in term 1. Re-sending same reply.
03:00:17.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.312374 4305 consensus_queue.cc:1048] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:00:17.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.367771 4348 sys_catalog.cc:455] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fdd32a3c604d4763b45610d82e3138c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:17.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.368196 4348 sys_catalog.cc:458] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: This master's current role is: FOLLOWER
03:00:17.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.368552 4341 sys_catalog.cc:455] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fdd32a3c604d4763b45610d82e3138c4. Latest consensus state: current_term: 1 leader_uuid: "fdd32a3c604d4763b45610d82e3138c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:17.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.368716 4341 sys_catalog.cc:458] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: This master's current role is: FOLLOWER
03:00:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.380028 4358 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:00:17.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.423187 4305 sys_catalog.cc:455] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fdd32a3c604d4763b45610d82e3138c4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:17.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.423568 4305 sys_catalog.cc:458] T 00000000000000000000000000000000 P fdd32a3c604d4763b45610d82e3138c4 [sys.catalog]: This master's current role is: LEADER
03:00:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.739789 4339 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:00:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.740146 4339 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:00:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.740247 4339 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:00:17.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.748884 4339 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:00:17.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.749030 4339 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:00:17.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.749076 4339 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:00:17.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.749110 4339 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:00:17.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.749187 4339 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.243.65
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.759104 4339 tablet_server_runner.cc:78] Tablet server non-default flags:
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/data
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/wal
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.65:0
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/data/info.pb
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.65
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.243.65
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/logs
03:00:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:00:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:00:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:00:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:00:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
03:00:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:00:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 02:43:21 UTC on 24a791456cd2
03:00:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9440
03:00:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:00:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.760699 4339 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:00:17.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.762579 4339 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:00:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.772570 4364 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.772800 4363 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:17.772840 4366 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.774224 4339 server_base.cc:1047] running on GCE node
03:00:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.774866 4339 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:00:17.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.775871 4339 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:00:17.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.777127 4339 hybrid_clock.cc:648] HybridClock initialized: now 1765335617777040 us; error 54 us; skew 500 ppm
03:00:17.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.777484 4339 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:00:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.779634 4339 webserver.cc:492] Webserver started at http://127.3.243.65:42553/ using document root <none> and password file <none>
03:00:17.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.780404 4339 fs_manager.cc:362] Metadata directory not provided
03:00:17.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.780532 4339 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:00:17.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.780902 4339 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
03:00:17.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.783319 4339 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/data/instance:
03:00:17.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bcf4e423f7b845e3ae355799a953cfa2"
03:00:17.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:17 on dist-test-slave-k7nm"
03:00:17.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.784067 4339 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/wal/instance:
03:00:17.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bcf4e423f7b845e3ae355799a953cfa2"
03:00:17.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:17 on dist-test-slave-k7nm"
03:00:17.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.789362 4339 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
03:00:17.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.792865 4372 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:00:17.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.794275 4339 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
03:00:17.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.794458 4339 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/data,/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/wal
03:00:17.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bcf4e423f7b845e3ae355799a953cfa2"
03:00:17.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:17 on dist-test-slave-k7nm"
03:00:17.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.794667 4339 fs_report.cc:389] FS layout report
03:00:17.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:00:17.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/wal
03:00:17.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/wal
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/data/data
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:00:17.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:00:17.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.843701 4339 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:00:17.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.844650 4339 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:00:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.844985 4339 kserver.cc:163] Server-wide thread pool size limit: 3276
03:00:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.848091 4339 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:00:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.848209 4339 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:00:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.848321 4339 ts_tablet_manager.cc:616] Registered 0 tablets
03:00:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.848423 4339 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:00:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.902058 4339 rpc_server.cc:307] RPC server started. Bound to: 127.3.243.65:36037
03:00:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.902104 4496 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.243.65:36037 every 8 connection(s)
03:00:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.903505 4339 server_base.cc:1179] Dumped server information to /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/data/info.pb
03:00:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.903982 4045 external_mini_cluster.cc:1428] Started /tmp/dist-test-task7KZUIc/build/asan/bin/kudu as pid 4339
03:00:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.904176 4045 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-0/wal/instance
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.908342 4498 heartbeater.cc:344] Connected to a master server at 127.3.243.126:33571
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.908736 4498 heartbeater.cc:461] Registering TS with master...
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.909381 4045 external_mini_cluster.cc:1366] Running /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/wal
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/data
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/logs
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/data/info.pb
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.66:0
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.243.66
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.66
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:00:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:00:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.909969 4498 heartbeater.cc:507] Master 127.3.243.126:33571 requested a full tablet report, sending...
03:00:17.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.912531 4086 ts_manager.cc:194] Registered new tserver with Master: bcf4e423f7b845e3ae355799a953cfa2 (127.3.243.65:36037)
03:00:17.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.916566 4499 heartbeater.cc:344] Connected to a master server at 127.3.243.125:39339
03:00:17.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.916702 4499 heartbeater.cc:461] Registering TS with master...
03:00:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.917146 4499 heartbeater.cc:507] Master 127.3.243.125:39339 requested a full tablet report, sending...
03:00:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.919119 4168 ts_manager.cc:194] Registered new tserver with Master: bcf4e423f7b845e3ae355799a953cfa2 (127.3.243.65:36037)
03:00:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.920603 4497 heartbeater.cc:344] Connected to a master server at 127.3.243.124:41589
03:00:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.920742 4497 heartbeater.cc:461] Registering TS with master...
03:00:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.921277 4497 heartbeater.cc:507] Master 127.3.243.124:41589 requested a full tablet report, sending...
03:00:17.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.922739 4249 ts_manager.cc:194] Registered new tserver with Master: bcf4e423f7b845e3ae355799a953cfa2 (127.3.243.65:36037)
03:00:17.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:17.924289 4249 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.243.65:50917
03:00:18.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.076610 4325 catalog_manager.cc:1269] Loaded cluster ID: cf189fddef194b5b91c14a3d072e7bc4
03:00:18.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.076715 4325 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038: loading cluster ID for follower catalog manager: success
03:00:18.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.079676 4325 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038: acquiring CA information for follower catalog manager: success
03:00:18.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.081872 4325 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:00:18.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.267889 4502 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:00:18.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.268213 4502 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:00:18.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.268306 4502 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:00:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.276583 4502 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:00:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.276723 4502 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:00:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.276793 4502 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:00:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.276842 4502 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:00:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.276925 4502 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.243.66
03:00:18.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.286604 4502 tablet_server_runner.cc:78] Tablet server non-default flags:
03:00:18.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:18.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:18.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:18.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:18.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/data
03:00:18.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/wal
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.66:0
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/data/info.pb
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.66
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.243.66
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/logs
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 02:43:21 UTC on 24a791456cd2
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9440
03:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:00:18.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.288147 4502 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:00:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.289943 4502 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:00:18.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.299865 4508 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:18.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.300289 4507 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:18.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.300716 4510 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:18.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.301201 4502 server_base.cc:1047] running on GCE node
03:00:18.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.301823 4502 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:00:18.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.302820 4502 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:00:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.304040 4502 hybrid_clock.cc:648] HybridClock initialized: now 1765335618303978 us; error 36 us; skew 500 ppm
03:00:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.304388 4502 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:00:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.306699 4502 webserver.cc:492] Webserver started at http://127.3.243.66:44859/ using document root <none> and password file <none>
03:00:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.307559 4502 fs_manager.cc:362] Metadata directory not provided
03:00:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.307685 4502 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:00:18.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.308023 4502 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
03:00:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.310492 4502 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/data/instance:
03:00:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e"
03:00:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:18 on dist-test-slave-k7nm"
03:00:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.311252 4502 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/wal/instance:
03:00:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e"
03:00:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:18 on dist-test-slave-k7nm"
03:00:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.316674 4502 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.001s
03:00:18.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.320015 4516 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:00:18.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.321440 4502 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
03:00:18.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.321599 4502 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/data,/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/wal
03:00:18.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e"
03:00:18.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:18 on dist-test-slave-k7nm"
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.321815 4502 fs_report.cc:389] FS layout report
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/wal
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/wal
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/data/data
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:00:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:00:18.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.342374 4502 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:00:18.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.343291 4502 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:00:18.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.343580 4502 kserver.cc:163] Server-wide thread pool size limit: 3276
03:00:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.346724 4502 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:00:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.346854 4502 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:00:18.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.346971 4502 ts_tablet_manager.cc:616] Registered 0 tablets
03:00:18.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.347071 4502 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:00:18.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.389518 4358 catalog_manager.cc:1269] Loaded cluster ID: cf189fddef194b5b91c14a3d072e7bc4
03:00:18.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.390095 4358 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173: loading cluster ID for follower catalog manager: success
03:00:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.392762 4358 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173: acquiring CA information for follower catalog manager: success
03:00:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.394768 4358 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:00:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.412948 4502 rpc_server.cc:307] RPC server started. Bound to: 127.3.243.66:43141
03:00:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.412962 4638 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.243.66:43141 every 8 connection(s)
03:00:18.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.414731 4502 server_base.cc:1179] Dumped server information to /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/data/info.pb
03:00:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.417532 4640 heartbeater.cc:344] Connected to a master server at 127.3.243.126:33571
03:00:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.418022 4640 heartbeater.cc:461] Registering TS with master...
03:00:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.419225 4640 heartbeater.cc:507] Master 127.3.243.126:33571 requested a full tablet report, sending...
03:00:18.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.421058 4086 ts_manager.cc:194] Registered new tserver with Master: 2bd9c23e31ad450f8cf481e0dfdf5f0e (127.3.243.66:43141)
03:00:18.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.424286 4045 external_mini_cluster.cc:1428] Started /tmp/dist-test-task7KZUIc/build/asan/bin/kudu as pid 4502
03:00:18.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.424503 4045 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-1/wal/instance
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.430042 4045 external_mini_cluster.cc:1366] Running /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7KZUIc/build/asan/bin/kudu
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/wal
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/data
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/logs
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/data/info.pb
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.67:0
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.243.67
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.67
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:00:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:00:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:00:18.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.439154 4641 heartbeater.cc:344] Connected to a master server at 127.3.243.125:39339
03:00:18.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.439333 4641 heartbeater.cc:461] Registering TS with master...
03:00:18.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.439824 4641 heartbeater.cc:507] Master 127.3.243.125:39339 requested a full tablet report, sending...
03:00:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.438529 4639 heartbeater.cc:344] Connected to a master server at 127.3.243.124:41589
03:00:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.440141 4639 heartbeater.cc:461] Registering TS with master...
03:00:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.440637 4639 heartbeater.cc:507] Master 127.3.243.124:41589 requested a full tablet report, sending...
03:00:18.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.441064 4168 ts_manager.cc:194] Registered new tserver with Master: 2bd9c23e31ad450f8cf481e0dfdf5f0e (127.3.243.66:43141)
03:00:18.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.441728 4249 ts_manager.cc:194] Registered new tserver with Master: 2bd9c23e31ad450f8cf481e0dfdf5f0e (127.3.243.66:43141)
03:00:18.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.442726 4249 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.243.66:60103
03:00:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.766693 4644 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:00:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.766981 4644 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:00:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.767072 4644 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:00:18.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.775625 4644 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:00:18.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.775760 4644 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:00:18.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.775808 4644 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:00:18.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.775842 4644 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:00:18.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.775918 4644 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.243.67
03:00:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.785686 4644 tablet_server_runner.cc:78] Tablet server non-default flags:
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.243.84:37285
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/data
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/wal
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.243.67:0
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/data/info.pb
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.243.67
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.243.126:33571,127.3.243.125:39339,127.3.243.124:41589
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.243.67
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/logs
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Dec 2025 02:43:21 UTC on 24a791456cd2
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9440
03:00:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
03:00:18.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.787536 4644 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:00:18.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.789367 4644 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:00:18.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.798878 4649 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:18.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.798902 4650 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:18.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.799161 4652 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:00:18.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.799711 4644 server_base.cc:1047] running on GCE node
03:00:18.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.800246 4644 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:00:18.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.801120 4644 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:00:18.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.802328 4644 hybrid_clock.cc:648] HybridClock initialized: now 1765335618802264 us; error 36 us; skew 500 ppm
03:00:18.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.802654 4644 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:00:18.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.804742 4644 webserver.cc:492] Webserver started at http://127.3.243.67:39737/ using document root <none> and password file <none>
03:00:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.805495 4644 fs_manager.cc:362] Metadata directory not provided
03:00:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.805591 4644 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:00:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.805873 4644 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
03:00:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.808180 4644 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/data/instance:
03:00:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04ef8b302cb34e32beaf73bdb49909ec"
03:00:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:18 on dist-test-slave-k7nm"
03:00:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.808928 4644 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/wal/instance:
03:00:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04ef8b302cb34e32beaf73bdb49909ec"
03:00:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:18 on dist-test-slave-k7nm"
03:00:18.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.814082 4644 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
03:00:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.817507 4658 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.818971 4644 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.819108 4644 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/data,/tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/wal
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04ef8b302cb34e32beaf73bdb49909ec"
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-10 03:00:18 on dist-test-slave-k7nm"
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.819343 4644 fs_report.cc:389] FS layout report
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/wal
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/wal
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/data/data
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:00:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:00:18.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.852610 4644 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:00:18.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.853566 4644 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:00:18.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.853814 4644 kserver.cc:163] Server-wide thread pool size limit: 3276
03:00:18.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.856844 4644 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:00:18.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.856963 4644 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
03:00:18.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.857064 4644 ts_tablet_manager.cc:616] Registered 0 tablets
03:00:18.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.857146 4644 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:00:18.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.917243 4644 rpc_server.cc:307] RPC server started. Bound to: 127.3.243.67:43563
03:00:18.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.917277 4779 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.243.67:43563 every 8 connection(s)
03:00:18.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.918589 4644 server_base.cc:1179] Dumped server information to /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/data/info.pb
03:00:18.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.921540 4781 heartbeater.cc:344] Connected to a master server at 127.3.243.126:33571
03:00:18.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.922883 4781 heartbeater.cc:461] Registering TS with master...
03:00:18.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.924260 4781 heartbeater.cc:507] Master 127.3.243.126:33571 requested a full tablet report, sending...
03:00:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.925170 4045 external_mini_cluster.cc:1428] Started /tmp/dist-test-task7KZUIc/build/asan/bin/kudu as pid 4644
03:00:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.925334 4045 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task7KZUIc/test-tmp/mini-kudu-cluster2993766745272537608/ts-2/wal/instance
03:00:18.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.925985 4086 ts_manager.cc:194] Registered new tserver with Master: 04ef8b302cb34e32beaf73bdb49909ec (127.3.243.67:43563)
03:00:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.926934 4497 heartbeater.cc:499] Master 127.3.243.124:41589 was elected leader, sending a full tablet report...
03:00:18.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.928265 4780 heartbeater.cc:344] Connected to a master server at 127.3.243.124:41589
03:00:18.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.928843 4780 heartbeater.cc:461] Registering TS with master...
03:00:18.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.929570 4780 heartbeater.cc:507] Master 127.3.243.124:41589 requested a full tablet report, sending...
03:00:18.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.930811 4782 heartbeater.cc:344] Connected to a master server at 127.3.243.125:39339
03:00:18.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.930872 4249 ts_manager.cc:194] Registered new tserver with Master: 04ef8b302cb34e32beaf73bdb49909ec (127.3.243.67:43563)
03:00:18.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.930958 4782 heartbeater.cc:461] Registering TS with master...
03:00:18.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.931375 4782 heartbeater.cc:507] Master 127.3.243.125:39339 requested a full tablet report, sending...
03:00:18.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.931700 4249 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.3.243.67:51127
03:00:18.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.932425 4168 ts_manager.cc:194] Registered new tserver with Master: 04ef8b302cb34e32beaf73bdb49909ec (127.3.243.67:43563)
03:00:18.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.936241 4045 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
03:00:18.938 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:00:18.938 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:00:18.939 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.243.126"
port: 33571
}
bound_http_address {
host: "127.3.243.126"
port: 46255
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.243.125"
port: 39339
}
bound_http_address {
host: "127.3.243.125"
port: 35657
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.243.124"
port: 41589
}
bound_http_address {
host: "127.3.243.124"
port: 36959
}
}
}
03:00:18.939 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:00:18.940 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.3.243.65"
port: 36037
}
bound_http_address {
host: "127.3.243.65"
port: 42553
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.3.243.66"
port: 43141
}
bound_http_address {
host: "127.3.243.66"
port: 44859
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.3.243.67"
port: 43563
}
bound_http_address {
host: "127.3.243.67"
port: 39737
}
}
}
03:00:18.940 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:00:18.942 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:00:18.943 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.243.125:39339(127.3.243.125:39339)] connecting to peer
03:00:18.944 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.243.126:33571(127.3.243.126:33571)] connecting to peer
03:00:18.945 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:18.945 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.243.125:39339(127.3.243.125:39339)] Successfully connected to peer
03:00:18.947 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.243.124:41589(127.3.243.124:41589)] Successfully connected to peer
03:00:18.946 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.3.243.126:33571(127.3.243.126:33571)] Successfully connected to peer
03:00:18.949 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.243.126
03:00:18.953 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.243.125
03:00:18.956 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.3.243.124
03:00:18.962 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x967d5f99, L:/127.0.0.1:57084 - R:/127.3.243.126:33571] using SASL/PLAIN
03:00:18.972 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9f6bfe63, L:/127.0.0.1:56024 - R:/127.3.243.125:39339] using SASL/PLAIN
03:00:18.973 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x44667391, L:/127.0.0.1:49926 - R:/127.3.243.124:41589] using SASL/PLAIN
03:00:18.975 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:00:18.975 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=32), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [20ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [30ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [31ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: OK, deferred=null) after lookup
03:00:18.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:18.977519 4249 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:49926:
03:00:18.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
03:00:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:18.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:18.979884 4249 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:00:19.009 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 6be52772103744a9931a2c9fb2f6c884
03:00:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.051489 4715 tablet_service.cc:1505] Processing CreateTablet for tablet e4dcf6f0fca540eeb668100804b36df1 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6be52772103744a9931a2c9fb2f6c884]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:00:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.051105 4432 tablet_service.cc:1505] Processing CreateTablet for tablet e4dcf6f0fca540eeb668100804b36df1 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6be52772103744a9931a2c9fb2f6c884]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:00:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.051868 4431 tablet_service.cc:1505] Processing CreateTablet for tablet c44e76cefd944f01b843f78eb75481d4 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6be52772103744a9931a2c9fb2f6c884]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:00:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.051105 4714 tablet_service.cc:1505] Processing CreateTablet for tablet c44e76cefd944f01b843f78eb75481d4 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6be52772103744a9931a2c9fb2f6c884]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:00:19.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.052700 4432 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4dcf6f0fca540eeb668100804b36df1. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:19.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.052711 4715 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4dcf6f0fca540eeb668100804b36df1. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:19.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.052472 4573 tablet_service.cc:1505] Processing CreateTablet for tablet c44e76cefd944f01b843f78eb75481d4 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6be52772103744a9931a2c9fb2f6c884]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:00:19.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.052802 4574 tablet_service.cc:1505] Processing CreateTablet for tablet e4dcf6f0fca540eeb668100804b36df1 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6be52772103744a9931a2c9fb2f6c884]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:00:19.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.053900 4573 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c44e76cefd944f01b843f78eb75481d4. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:19.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.054006 4431 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c44e76cefd944f01b843f78eb75481d4. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:19.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.054937 4574 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e4dcf6f0fca540eeb668100804b36df1. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:19.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.056522 4714 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c44e76cefd944f01b843f78eb75481d4. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:19.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.082551 4807 tablet_bootstrap.cc:492] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec: Bootstrap starting.
03:00:19.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.084049 4805 tablet_bootstrap.cc:492] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2: Bootstrap starting.
03:00:19.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.084828 4806 tablet_bootstrap.cc:492] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Bootstrap starting.
03:00:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.087596 4807 tablet_bootstrap.cc:654] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec: Neither blocks nor log segments found. Creating new log.
03:00:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.087705 4806 tablet_bootstrap.cc:654] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Neither blocks nor log segments found. Creating new log.
03:00:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.087853 4805 tablet_bootstrap.cc:654] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2: Neither blocks nor log segments found. Creating new log.
03:00:19.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.088860 4806 log.cc:826] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Log is configured to *not* fsync() on all Append() calls
03:00:19.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.089020 4807 log.cc:826] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec: Log is configured to *not* fsync() on all Append() calls
03:00:19.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.089022 4805 log.cc:826] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2: Log is configured to *not* fsync() on all Append() calls
03:00:19.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.091650 4807 tablet_bootstrap.cc:492] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec: No bootstrap required, opened a new log
03:00:19.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.091784 4806 tablet_bootstrap.cc:492] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: No bootstrap required, opened a new log
03:00:19.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.091776 4805 tablet_bootstrap.cc:492] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2: No bootstrap required, opened a new log
03:00:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.091953 4807 ts_tablet_manager.cc:1403] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec: Time spent bootstrapping tablet: real 0.010s user 0.000s sys 0.007s
03:00:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.092031 4805 ts_tablet_manager.cc:1403] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.000s
03:00:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.092036 4806 ts_tablet_manager.cc:1403] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.000s
03:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.097275 4807 raft_consensus.cc:359] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } }
03:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.097631 4807 raft_consensus.cc:385] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.097725 4807 raft_consensus.cc:740] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 04ef8b302cb34e32beaf73bdb49909ec, State: Initialized, Role: FOLLOWER
03:00:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.098384 4807 consensus_queue.cc:260] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } }
03:00:19.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.099154 4806 raft_consensus.cc:359] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.099519 4780 heartbeater.cc:499] Master 127.3.243.124:41589 was elected leader, sending a full tablet report...
03:00:19.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.099754 4806 raft_consensus.cc:385] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:19.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.099547 4805 raft_consensus.cc:359] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } }
03:00:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.099916 4805 raft_consensus.cc:385] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.100016 4805 raft_consensus.cc:740] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bcf4e423f7b845e3ae355799a953cfa2, State: Initialized, Role: FOLLOWER
03:00:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.100057 4806 raft_consensus.cc:740] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2bd9c23e31ad450f8cf481e0dfdf5f0e, State: Initialized, Role: FOLLOWER
03:00:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.100186 4807 ts_tablet_manager.cc:1434] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec: Time spent starting tablet: real 0.008s user 0.008s sys 0.001s
03:00:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.100821 4805 consensus_queue.cc:260] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } }
03:00:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.101233 4807 tablet_bootstrap.cc:492] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec: Bootstrap starting.
03:00:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.102933 4807 tablet_bootstrap.cc:654] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec: Neither blocks nor log segments found. Creating new log.
03:00:19.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.115162 4806 consensus_queue.cc:260] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.117110 4807 tablet_bootstrap.cc:492] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec: No bootstrap required, opened a new log
03:00:19.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.117307 4807 ts_tablet_manager.cc:1403] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec: Time spent bootstrapping tablet: real 0.017s user 0.004s sys 0.008s
03:00:19.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.118186 4807 raft_consensus.cc:359] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.118393 4807 raft_consensus.cc:385] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:19.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.118481 4807 raft_consensus.cc:740] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 04ef8b302cb34e32beaf73bdb49909ec, State: Initialized, Role: FOLLOWER
03:00:19.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.119681 4812 raft_consensus.cc:493] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:00:19.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.119917 4812 raft_consensus.cc:515] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } }
03:00:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.122191 4805 ts_tablet_manager.cc:1434] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2: Time spent starting tablet: real 0.030s user 0.016s sys 0.007s
03:00:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.122843 4807 consensus_queue.cc:260] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.123421 4807 ts_tablet_manager.cc:1434] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
03:00:19.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.123611 4812 leader_election.cc:290] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 04ef8b302cb34e32beaf73bdb49909ec (127.3.243.67:43563), 2bd9c23e31ad450f8cf481e0dfdf5f0e (127.3.243.66:43141)
03:00:19.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.123826 4805 tablet_bootstrap.cc:492] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2: Bootstrap starting.
03:00:19.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.125607 4805 tablet_bootstrap.cc:654] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2: Neither blocks nor log segments found. Creating new log.
03:00:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.127676 4805 tablet_bootstrap.cc:492] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2: No bootstrap required, opened a new log
03:00:19.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.127821 4639 heartbeater.cc:499] Master 127.3.243.124:41589 was elected leader, sending a full tablet report...
03:00:19.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.127858 4805 ts_tablet_manager.cc:1403] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
03:00:19.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.128721 4805 raft_consensus.cc:359] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.128926 4805 raft_consensus.cc:385] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:19.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.129017 4805 raft_consensus.cc:740] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bcf4e423f7b845e3ae355799a953cfa2, State: Initialized, Role: FOLLOWER
03:00:19.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.129232 4805 consensus_queue.cc:260] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.129765 4805 ts_tablet_manager.cc:1434] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
03:00:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.136490 4806 ts_tablet_manager.cc:1434] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Time spent starting tablet: real 0.044s user 0.013s sys 0.000s
03:00:19.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.138754 4806 tablet_bootstrap.cc:492] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Bootstrap starting.
03:00:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.140936 4806 tablet_bootstrap.cc:654] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Neither blocks nor log segments found. Creating new log.
03:00:19.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.143533 4806 tablet_bootstrap.cc:492] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: No bootstrap required, opened a new log
03:00:19.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.143731 4806 ts_tablet_manager.cc:1403] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.002s
03:00:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.144667 4735 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "e4dcf6f0fca540eeb668100804b36df1" candidate_uuid: "bcf4e423f7b845e3ae355799a953cfa2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04ef8b302cb34e32beaf73bdb49909ec" is_pre_election: true
03:00:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.144799 4806 raft_consensus.cc:359] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } }
03:00:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.144970 4806 raft_consensus.cc:385] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.145051 4806 raft_consensus.cc:740] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2bd9c23e31ad450f8cf481e0dfdf5f0e, State: Initialized, Role: FOLLOWER
03:00:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.145237 4735 raft_consensus.cc:2468] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bcf4e423f7b845e3ae355799a953cfa2 in term 0.
03:00:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.145263 4806 consensus_queue.cc:260] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } }
03:00:19.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.145910 4375 leader_election.cc:304] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 04ef8b302cb34e32beaf73bdb49909ec, bcf4e423f7b845e3ae355799a953cfa2; no voters:
03:00:19.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.146358 4814 raft_consensus.cc:2804] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:00:19.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.146514 4806 ts_tablet_manager.cc:1434] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
03:00:19.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.146505 4814 raft_consensus.cc:493] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:00:19.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.146633 4814 raft_consensus.cc:3060] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Advancing to term 1
03:00:19.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.146732 4594 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "e4dcf6f0fca540eeb668100804b36df1" candidate_uuid: "bcf4e423f7b845e3ae355799a953cfa2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" is_pre_election: true
03:00:19.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.147222 4594 raft_consensus.cc:2468] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bcf4e423f7b845e3ae355799a953cfa2 in term 0.
03:00:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.149245 4814 raft_consensus.cc:515] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } }
03:00:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.150343 4735 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "e4dcf6f0fca540eeb668100804b36df1" candidate_uuid: "bcf4e423f7b845e3ae355799a953cfa2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04ef8b302cb34e32beaf73bdb49909ec"
03:00:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.150538 4735 raft_consensus.cc:3060] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Advancing to term 1
03:00:19.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.151094 4814 leader_election.cc:290] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [CANDIDATE]: Term 1 election: Requested vote from peers 04ef8b302cb34e32beaf73bdb49909ec (127.3.243.67:43563), 2bd9c23e31ad450f8cf481e0dfdf5f0e (127.3.243.66:43141)
03:00:19.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.151516 4594 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "e4dcf6f0fca540eeb668100804b36df1" candidate_uuid: "bcf4e423f7b845e3ae355799a953cfa2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e"
03:00:19.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.151680 4594 raft_consensus.cc:3060] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Advancing to term 1
03:00:19.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.152858 4735 raft_consensus.cc:2468] T e4dcf6f0fca540eeb668100804b36df1 P 04ef8b302cb34e32beaf73bdb49909ec [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bcf4e423f7b845e3ae355799a953cfa2 in term 1.
03:00:19.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.153232 4375 leader_election.cc:304] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 04ef8b302cb34e32beaf73bdb49909ec, bcf4e423f7b845e3ae355799a953cfa2; no voters:
03:00:19.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.153476 4814 raft_consensus.cc:2804] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [term 1 FOLLOWER]: Leader election won for term 1
03:00:19.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.153652 4814 raft_consensus.cc:697] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [term 1 LEADER]: Becoming Leader. State: Replica: bcf4e423f7b845e3ae355799a953cfa2, State: Running, Role: LEADER
03:00:19.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.153908 4594 raft_consensus.cc:2468] T e4dcf6f0fca540eeb668100804b36df1 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bcf4e423f7b845e3ae355799a953cfa2 in term 1.
03:00:19.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.154166 4814 consensus_queue.cc:237] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } }
03:00:19.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.159368 4249 catalog_manager.cc:5654] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 reported cstate change: term changed from 0 to 1, leader changed from <none> to bcf4e423f7b845e3ae355799a953cfa2 (127.3.243.65). New cstate: current_term: 1 leader_uuid: "bcf4e423f7b845e3ae355799a953cfa2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } health_report { overall_health: UNKNOWN } } }
03:00:19.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:19.160614 4500 tablet.cc:2378] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:00:19.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:19.172520 4783 tablet.cc:2378] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:00:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:19.190150 4642 tablet.cc:2378] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:00:19.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.203238 4811 raft_consensus.cc:493] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:00:19.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.203466 4811 raft_consensus.cc:515] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.205277 4811 leader_election.cc:290] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bcf4e423f7b845e3ae355799a953cfa2 (127.3.243.65:36037), 2bd9c23e31ad450f8cf481e0dfdf5f0e (127.3.243.66:43141)
03:00:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.211158 4594 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "c44e76cefd944f01b843f78eb75481d4" candidate_uuid: "04ef8b302cb34e32beaf73bdb49909ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" is_pre_election: true
03:00:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.211303 4452 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "c44e76cefd944f01b843f78eb75481d4" candidate_uuid: "04ef8b302cb34e32beaf73bdb49909ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bcf4e423f7b845e3ae355799a953cfa2" is_pre_election: true
03:00:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.211452 4594 raft_consensus.cc:2468] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 04ef8b302cb34e32beaf73bdb49909ec in term 0.
03:00:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.211712 4452 raft_consensus.cc:2468] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 04ef8b302cb34e32beaf73bdb49909ec in term 0.
03:00:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.211961 4660 leader_election.cc:304] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 04ef8b302cb34e32beaf73bdb49909ec, 2bd9c23e31ad450f8cf481e0dfdf5f0e; no voters:
03:00:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.212340 4811 raft_consensus.cc:2804] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Leader pre-election won for term 1
03:00:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.212473 4811 raft_consensus.cc:493] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:00:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.212594 4811 raft_consensus.cc:3060] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Advancing to term 1
03:00:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.214510 4811 raft_consensus.cc:515] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.215077 4811 leader_election.cc:290] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [CANDIDATE]: Term 1 election: Requested vote from peers bcf4e423f7b845e3ae355799a953cfa2 (127.3.243.65:36037), 2bd9c23e31ad450f8cf481e0dfdf5f0e (127.3.243.66:43141)
03:00:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.215588 4452 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "c44e76cefd944f01b843f78eb75481d4" candidate_uuid: "04ef8b302cb34e32beaf73bdb49909ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bcf4e423f7b845e3ae355799a953cfa2"
03:00:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.215600 4594 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "c44e76cefd944f01b843f78eb75481d4" candidate_uuid: "04ef8b302cb34e32beaf73bdb49909ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e"
03:00:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.215745 4452 raft_consensus.cc:3060] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Advancing to term 1
03:00:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.215824 4594 raft_consensus.cc:3060] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Advancing to term 1
03:00:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.217661 4452 raft_consensus.cc:2468] T c44e76cefd944f01b843f78eb75481d4 P bcf4e423f7b845e3ae355799a953cfa2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 04ef8b302cb34e32beaf73bdb49909ec in term 1.
03:00:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.217664 4594 raft_consensus.cc:2468] T c44e76cefd944f01b843f78eb75481d4 P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 04ef8b302cb34e32beaf73bdb49909ec in term 1.
03:00:19.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.218010 4660 leader_election.cc:304] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 04ef8b302cb34e32beaf73bdb49909ec, bcf4e423f7b845e3ae355799a953cfa2; no voters:
03:00:19.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.218286 4811 raft_consensus.cc:2804] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [term 1 FOLLOWER]: Leader election won for term 1
03:00:19.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.218675 4811 raft_consensus.cc:697] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [term 1 LEADER]: Becoming Leader. State: Replica: 04ef8b302cb34e32beaf73bdb49909ec, State: Running, Role: LEADER
03:00:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.219398 4811 consensus_queue.cc:237] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.224699 4249 catalog_manager.cc:5654] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec reported cstate change: term changed from 0 to 1, leader changed from <none> to 04ef8b302cb34e32beaf73bdb49909ec (127.3.243.67). New cstate: current_term: 1 leader_uuid: "04ef8b302cb34e32beaf73bdb49909ec" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } health_report { overall_health: HEALTHY } } }
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.363833 4249 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:49942:
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:00:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:19.364898 4249 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:00:19.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.377202 4432 tablet_service.cc:1505] Processing CreateTablet for tablet fc3ecc41943943d9ad97893bab2956ef (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=6617aa2658c441eeaf45d6297335615b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:00:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.377199 4715 tablet_service.cc:1505] Processing CreateTablet for tablet fc3ecc41943943d9ad97893bab2956ef (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=6617aa2658c441eeaf45d6297335615b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:00:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.378124 4715 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fc3ecc41943943d9ad97893bab2956ef. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.378149 4432 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fc3ecc41943943d9ad97893bab2956ef. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.383098 4807 tablet_bootstrap.cc:492] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: Bootstrap starting.
03:00:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.383098 4805 tablet_bootstrap.cc:492] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2: Bootstrap starting.
03:00:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.383623 4573 tablet_service.cc:1505] Processing CreateTablet for tablet fc3ecc41943943d9ad97893bab2956ef (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=6617aa2658c441eeaf45d6297335615b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:00:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.384590 4807 tablet_bootstrap.cc:654] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: Neither blocks nor log segments found. Creating new log.
03:00:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.384613 4573 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fc3ecc41943943d9ad97893bab2956ef. 1 dirs total, 0 dirs full, 0 dirs failed
03:00:19.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.384744 4805 tablet_bootstrap.cc:654] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2: Neither blocks nor log segments found. Creating new log.
03:00:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.386597 4805 tablet_bootstrap.cc:492] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2: No bootstrap required, opened a new log
03:00:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.386754 4805 ts_tablet_manager.cc:1403] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
03:00:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.386976 4807 tablet_bootstrap.cc:492] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: No bootstrap required, opened a new log
03:00:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.387087 4807 ts_tablet_manager.cc:1403] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.002s
03:00:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.387463 4805 raft_consensus.cc:359] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.387647 4805 raft_consensus.cc:385] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.387630 4807 raft_consensus.cc:359] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.387717 4805 raft_consensus.cc:740] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bcf4e423f7b845e3ae355799a953cfa2, State: Initialized, Role: FOLLOWER
03:00:19.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.387770 4807 raft_consensus.cc:385] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:19.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.387813 4807 raft_consensus.cc:740] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 04ef8b302cb34e32beaf73bdb49909ec, State: Initialized, Role: FOLLOWER
03:00:19.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.388022 4807 consensus_queue.cc:260] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.387962 4805 consensus_queue.cc:260] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.388490 4807 ts_tablet_manager.cc:1434] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
03:00:19.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.388438 4811 tablet_replica.cc:442] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } }
03:00:19.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.388653 4811 tablet_replica.cc:445] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: This TxnStatusTablet replica's current role is: FOLLOWER
03:00:19.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.391364 4814 tablet_replica.cc:442] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } }
03:00:19.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.391505 4806 tablet_bootstrap.cc:492] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Bootstrap starting.
03:00:19.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.391584 4814 tablet_replica.cc:445] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2: This TxnStatusTablet replica's current role is: FOLLOWER
03:00:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.393216 4806 tablet_bootstrap.cc:654] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Neither blocks nor log segments found. Creating new log.
03:00:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.393260 4805 ts_tablet_manager.cc:1434] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
03:00:19.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.398191 4806 tablet_bootstrap.cc:492] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e: No bootstrap required, opened a new log
03:00:19.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.398367 4806 ts_tablet_manager.cc:1403] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Time spent bootstrapping tablet: real 0.007s user 0.004s sys 0.000s
03:00:19.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.398949 4806 raft_consensus.cc:359] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.399102 4806 raft_consensus.cc:385] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:00:19.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.399163 4806 raft_consensus.cc:740] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2bd9c23e31ad450f8cf481e0dfdf5f0e, State: Initialized, Role: FOLLOWER
03:00:19.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.399372 4806 consensus_queue.cc:260] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.399816 4815 tablet_replica.cc:442] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } }
03:00:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.400167 4815 tablet_replica.cc:445] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e: This TxnStatusTablet replica's current role is: FOLLOWER
03:00:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.399819 4806 ts_tablet_manager.cc:1434] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
03:00:19.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.556833 4814 consensus_queue.cc:1048] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:00:19.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.568076 4814 consensus_queue.cc:1048] T e4dcf6f0fca540eeb668100804b36df1 P bcf4e423f7b845e3ae355799a953cfa2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:00:19.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.611948 4811 consensus_queue.cc:1048] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [LEADER]: Connected to new peer: Peer: permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:00:19.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.617290 4822 raft_consensus.cc:493] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:00:19.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.617457 4822 raft_consensus.cc:515] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.618070 4822 leader_election.cc:290] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bcf4e423f7b845e3ae355799a953cfa2 (127.3.243.65:36037), 2bd9c23e31ad450f8cf481e0dfdf5f0e (127.3.243.66:43141)
03:00:19.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.618566 4594 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "fc3ecc41943943d9ad97893bab2956ef" candidate_uuid: "04ef8b302cb34e32beaf73bdb49909ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" is_pre_election: true
03:00:19.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.618782 4594 raft_consensus.cc:2468] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 04ef8b302cb34e32beaf73bdb49909ec in term 0.
03:00:19.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.619235 4660 leader_election.cc:304] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 04ef8b302cb34e32beaf73bdb49909ec, 2bd9c23e31ad450f8cf481e0dfdf5f0e; no voters:
03:00:19.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.619637 4822 raft_consensus.cc:2804] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Leader pre-election won for term 1
03:00:19.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.619750 4822 raft_consensus.cc:493] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:00:19.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.619843 4822 raft_consensus.cc:3060] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [term 0 FOLLOWER]: Advancing to term 1
03:00:19.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.619822 4811 consensus_queue.cc:1048] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec [LEADER]: Connected to new peer: Peer: permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:00:19.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.620435 4452 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "fc3ecc41943943d9ad97893bab2956ef" candidate_uuid: "04ef8b302cb34e32beaf73bdb49909ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bcf4e423f7b845e3ae355799a953cfa2" is_pre_election: true
03:00:19.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.620687 4452 raft_consensus.cc:2468] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 04ef8b302cb34e32beaf73bdb49909ec in term 0.
03:00:19.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.622529 4822 raft_consensus.cc:515] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.623658 4822 leader_election.cc:290] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [CANDIDATE]: Term 1 election: Requested vote from peers bcf4e423f7b845e3ae355799a953cfa2 (127.3.243.65:36037), 2bd9c23e31ad450f8cf481e0dfdf5f0e (127.3.243.66:43141)
03:00:19.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.623948 4594 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "fc3ecc41943943d9ad97893bab2956ef" candidate_uuid: "04ef8b302cb34e32beaf73bdb49909ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e"
03:00:19.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.624233 4594 raft_consensus.cc:3060] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 0 FOLLOWER]: Advancing to term 1
03:00:19.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.624284 4452 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "fc3ecc41943943d9ad97893bab2956ef" candidate_uuid: "04ef8b302cb34e32beaf73bdb49909ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bcf4e423f7b845e3ae355799a953cfa2"
03:00:19.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.624486 4452 raft_consensus.cc:3060] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2 [term 0 FOLLOWER]: Advancing to term 1
03:00:19.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.627310 4452 raft_consensus.cc:2468] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 04ef8b302cb34e32beaf73bdb49909ec in term 1.
03:00:19.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.627331 4594 raft_consensus.cc:2468] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 04ef8b302cb34e32beaf73bdb49909ec in term 1.
03:00:19.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.627744 4660 leader_election.cc:304] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 04ef8b302cb34e32beaf73bdb49909ec, bcf4e423f7b845e3ae355799a953cfa2; no voters:
03:00:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.628293 4822 raft_consensus.cc:2804] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [term 1 FOLLOWER]: Leader election won for term 1
03:00:19.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.629401 4822 raft_consensus.cc:697] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [term 1 LEADER]: Becoming Leader. State: Replica: 04ef8b302cb34e32beaf73bdb49909ec, State: Running, Role: LEADER
03:00:19.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.629766 4822 consensus_queue.cc:237] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } }
03:00:19.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.632074 4811 tablet_replica.cc:442] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: TxnStatusTablet state changed. Reason: New leader 04ef8b302cb34e32beaf73bdb49909ec. Latest consensus state: current_term: 1 leader_uuid: "04ef8b302cb34e32beaf73bdb49909ec" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } }
03:00:19.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.632292 4811 tablet_replica.cc:445] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: This TxnStatusTablet replica's current role is: LEADER
03:00:19.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.633296 4249 catalog_manager.cc:5654] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec reported cstate change: term changed from 0 to 1, leader changed from <none> to 04ef8b302cb34e32beaf73bdb49909ec (127.3.243.67). New cstate: current_term: 1 leader_uuid: "04ef8b302cb34e32beaf73bdb49909ec" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } health_report { overall_health: HEALTHY } } }
03:00:19.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.633721 4844 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
03:00:19.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.633977 4844 txn_status_manager.cc:930] Loading transaction status metadata into memory...
03:00:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.702919 4452 raft_consensus.cc:1275] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2 [term 1 FOLLOWER]: Refusing update from remote peer 04ef8b302cb34e32beaf73bdb49909ec: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:00:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.702905 4594 raft_consensus.cc:1275] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e [term 1 FOLLOWER]: Refusing update from remote peer 04ef8b302cb34e32beaf73bdb49909ec: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:00:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.703524 4811 consensus_queue.cc:1048] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [LEADER]: Connected to new peer: Peer: permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:00:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.703792 4822 consensus_queue.cc:1048] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec [LEADER]: Connected to new peer: Peer: permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:00:19.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.716625 4815 tablet_replica.cc:442] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e: TxnStatusTablet state changed. Reason: New leader 04ef8b302cb34e32beaf73bdb49909ec. Latest consensus state: current_term: 1 leader_uuid: "04ef8b302cb34e32beaf73bdb49909ec" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } }
03:00:19.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.716902 4815 tablet_replica.cc:445] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e: This TxnStatusTablet replica's current role is: FOLLOWER
03:00:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.720342 4837 mvcc.cc:204] Tried to move back new op lower bound from 7230814698295156736 to 7230814698006622208. Current Snapshot: MvccSnapshot[applied={T|T < 7230814698295156736}]
03:00:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.721453 4839 mvcc.cc:204] Tried to move back new op lower bound from 7230814698295156736 to 7230814698006622208. Current Snapshot: MvccSnapshot[applied={T|T < 7230814698295156736}]
03:00:19.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.722208 4811 tablet_replica.cc:442] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "04ef8b302cb34e32beaf73bdb49909ec" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } }
03:00:19.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.722406 4811 tablet_replica.cc:445] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: This TxnStatusTablet replica's current role is: LEADER
03:00:19.728 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
03:00:19.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.730126 4815 tablet_replica.cc:442] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "04ef8b302cb34e32beaf73bdb49909ec" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } }
03:00:19.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.730353 4815 tablet_replica.cc:445] T fc3ecc41943943d9ad97893bab2956ef P 2bd9c23e31ad450f8cf481e0dfdf5f0e: This TxnStatusTablet replica's current role is: FOLLOWER
03:00:19.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.731631 4814 tablet_replica.cc:442] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2: TxnStatusTablet state changed. Reason: New leader 04ef8b302cb34e32beaf73bdb49909ec. Latest consensus state: current_term: 1 leader_uuid: "04ef8b302cb34e32beaf73bdb49909ec" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } }
03:00:19.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.731855 4814 tablet_replica.cc:445] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2: This TxnStatusTablet replica's current role is: FOLLOWER
03:00:19.734 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 6be52772103744a9931a2c9fb2f6c884
03:00:19.735 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.243.124:41589
03:00:19.735 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:00:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.736127 4841 tablet_replica.cc:442] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "04ef8b302cb34e32beaf73bdb49909ec" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } }
03:00:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.736488 4841 tablet_replica.cc:445] T fc3ecc41943943d9ad97893bab2956ef P 04ef8b302cb34e32beaf73bdb49909ec: This TxnStatusTablet replica's current role is: LEADER
03:00:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.740281 4045 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task7KZUIc/build/asan/bin/kudu pid 4215
03:00:19.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.741700 4289 generic_service.cc:196] Checking for leaks (request via RPC)
03:00:19.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.744055 4814 tablet_replica.cc:442] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "04ef8b302cb34e32beaf73bdb49909ec" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bcf4e423f7b845e3ae355799a953cfa2" member_type: VOTER last_known_addr { host: "127.3.243.65" port: 36037 } } peers { permanent_uuid: "2bd9c23e31ad450f8cf481e0dfdf5f0e" member_type: VOTER last_known_addr { host: "127.3.243.66" port: 43141 } } peers { permanent_uuid: "04ef8b302cb34e32beaf73bdb49909ec" member_type: VOTER last_known_addr { host: "127.3.243.67" port: 43563 } } }
03:00:19.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.744277 4814 tablet_replica.cc:445] T fc3ecc41943943d9ad97893bab2956ef P bcf4e423f7b845e3ae355799a953cfa2: This TxnStatusTablet replica's current role is: FOLLOWER
03:00:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:19.866539 4045 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task7KZUIc/build/asan/bin/kudu with pid 4215
03:00:19.885 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.243.124:41589(127.3.243.124:41589)] handling channelInactive
03:00:19.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state READY due to: connection closed
03:00:19.886 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:00:19.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:19.973024 4497 heartbeater.cc:646] Failed to heartbeat to 127.3.243.124:41589 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.091517 4780 heartbeater.cc:646] Failed to heartbeat to 127.3.243.124:41589 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.167181 4857 raft_consensus.cc:493] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fdd32a3c604d4763b45610d82e3138c4)
03:00:20.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.167182 4858 raft_consensus.cc:493] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fdd32a3c604d4763b45610d82e3138c4)
03:00:20.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.167403 4858 raft_consensus.cc:515] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:20.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.167414 4857 raft_consensus.cc:515] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:20.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.168934 4858 leader_election.cc:290] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 28b4d256f6804d7aab5e92586dc1b038 (127.3.243.126:33571), fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589)
03:00:20.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.169406 4096 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b98d89b2a2444f97bab4ad389ef0c173" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "28b4d256f6804d7aab5e92586dc1b038" is_pre_election: true
03:00:20.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.169661 4096 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b98d89b2a2444f97bab4ad389ef0c173 in term 1.
03:00:20.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.170089 4857 leader_election.cc:290] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b98d89b2a2444f97bab4ad389ef0c173 (127.3.243.125:39339), fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589)
03:00:20.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.170404 4178 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "28b4d256f6804d7aab5e92586dc1b038" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b98d89b2a2444f97bab4ad389ef0c173" is_pre_election: true
03:00:20.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.170634 4178 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 28b4d256f6804d7aab5e92586dc1b038 in term 1.
03:00:20.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.171082 4154 leader_election.cc:304] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 28b4d256f6804d7aab5e92586dc1b038, b98d89b2a2444f97bab4ad389ef0c173; no voters:
03:00:20.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.171284 4071 leader_election.cc:304] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 28b4d256f6804d7aab5e92586dc1b038, b98d89b2a2444f97bab4ad389ef0c173; no voters:
03:00:20.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.171538 4858 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 1 FOLLOWER]: Leader pre-election won for term 2
03:00:20.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.171653 4858 raft_consensus.cc:493] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 1 FOLLOWER]: Starting leader election (detected failure of leader fdd32a3c604d4763b45610d82e3138c4)
03:00:20.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.171764 4858 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 1 FOLLOWER]: Advancing to term 2
03:00:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.171950 4857 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 1 FOLLOWER]: Leader pre-election won for term 2
03:00:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.172062 4857 raft_consensus.cc:493] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 1 FOLLOWER]: Starting leader election (detected failure of leader fdd32a3c604d4763b45610d82e3138c4)
03:00:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.172161 4857 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 1 FOLLOWER]: Advancing to term 2
03:00:20.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.173092 4073 leader_election.cc:336] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.173700 4154 leader_election.cc:336] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.174322 4858 raft_consensus.cc:515] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:20.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.174571 4857 raft_consensus.cc:515] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:20.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.174831 4858 leader_election.cc:290] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 2 election: Requested vote from peers 28b4d256f6804d7aab5e92586dc1b038 (127.3.243.126:33571), fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589)
03:00:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.175179 4096 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b98d89b2a2444f97bab4ad389ef0c173" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "28b4d256f6804d7aab5e92586dc1b038"
03:00:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.175452 4096 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate b98d89b2a2444f97bab4ad389ef0c173 in current term 2: Already voted for candidate 28b4d256f6804d7aab5e92586dc1b038 in this term.
03:00:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.175586 4857 leader_election.cc:290] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 2 election: Requested vote from peers b98d89b2a2444f97bab4ad389ef0c173 (127.3.243.125:39339), fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589)
03:00:20.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.176255 4154 leader_election.cc:336] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.176257 4178 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "28b4d256f6804d7aab5e92586dc1b038" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b98d89b2a2444f97bab4ad389ef0c173"
03:00:20.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.176417 4154 leader_election.cc:304] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b98d89b2a2444f97bab4ad389ef0c173; no voters: 28b4d256f6804d7aab5e92586dc1b038, fdd32a3c604d4763b45610d82e3138c4
03:00:20.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.176528 4178 raft_consensus.cc:2393] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 28b4d256f6804d7aab5e92586dc1b038 in current term 2: Already voted for candidate b98d89b2a2444f97bab4ad389ef0c173 in this term.
03:00:20.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.177078 4858 raft_consensus.cc:2749] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
03:00:20.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.177333 4073 leader_election.cc:336] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.177472 4073 leader_election.cc:304] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 28b4d256f6804d7aab5e92586dc1b038; no voters: b98d89b2a2444f97bab4ad389ef0c173, fdd32a3c604d4763b45610d82e3138c4
03:00:20.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.177763 4857 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
03:00:20.244 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.245 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.245 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.3.243.124:41589(127.3.243.124:41589) for tablet Kudu Master: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.264 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.266 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.283 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.285 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.304 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.305 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.305 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.305 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.323 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.324 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.325 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.343 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.344 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.404 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.406 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.444 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.446 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.464 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.465 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.466 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.467 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.3.243.124:41589, [1ms] received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, deferred=Deferred@1904520819(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$120/48695226@34bd48c9 -> (continuation of Deferred@329219607 after retry RPC after error@468862709) -> (continuation of Deferred@1330742478 after retry RPC after error@292845293) -> (continuation of Deferred@2037632274 after retry RPC after error@1883240743) -> (continuation of Deferred@864777242 after retry RPC after error@24821656) -> (continuation of Deferred@1711729144 after retry RPC after error@1769980638) -> (continuation of Deferred@29114447 after retry RPC after error@1433315743) -> (continuation of Deferred@83284963 after retry RPC after error@319326353), errback=org.apache.kudu.client.KuduTransaction$$Lambda$121/518121423@70d51c87 -> (continuation of Deferred@329219607 after retry RPC after error@468862709) -> (continuation of Deferred@1330742478 after retry RPC after error@292845293) -> (continuation of Deferred@2037632274 after retry RPC after error@1883240743) -> (continuation of Deferred@864777242 after retry RPC after error@24821656) -> (continuation of Deferred@1711729144 after retry RPC after error@1769980638) -> (continuation of Deferred@29114447 after retry RPC after error@1433315743) -> (continuation of Deferred@83284963 after retry RPC after error@319326353)))
03:00:20.467 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.3.243.124:41589, [1ms] received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [201ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [202ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [221ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, deferred=Deferred@1904520819(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$120/48695226@34bd48c9 -> (continuation of Deferred@329219607 after retry RPC after error@468862709) -> (continuation of Deferred@1330742478 after retry RPC after error@292845293) -> (continuation of Deferred@2037632274 after retry RPC after error@1883240743) -> (continuation of Deferred@864777242 after retry RPC after error@24821656) -> (continuation of Deferred@1711729144 after retry RPC after error@1769980638) -> (continuation of Deferred@29114447 after retry RPC after error@1433315743) -> (continuation of Deferred@83284963 after retry RPC after error@319326353), errback=org.apache.kudu.client.KuduTransaction$$Lambda$121/518121423@70d51c87 -> (continuation of Deferred@329219607 after retry RPC after error@468862709) -> (continuation of Deferred@1330742478 after retry RPC after error@292845293) -> (continuation of Deferred@2037632274 after retry RPC after error@1883240743) -> (continuation of Deferred@864777242 after retry RPC after error@24821656) -> (continuation of Deferred@1711729144 after retry RPC after error@1769980638) -> (continuation of Deferred@29114447 after retry RPC after error@1433315743) -> (continuation of Deferred@83284963 after retry RPC after error@319326353)))
03:00:20.484 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.486 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.504 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.506 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.506 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.524 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.525 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.544 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.546 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.564 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.584 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.586 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.600958 4857 raft_consensus.cc:493] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:00:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.600973 4858 raft_consensus.cc:493] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:00:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.601152 4858 raft_consensus.cc:515] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.601189 4857 raft_consensus.cc:515] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.601739 4858 leader_election.cc:290] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 28b4d256f6804d7aab5e92586dc1b038 (127.3.243.126:33571), fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589)
03:00:20.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.601814 4857 leader_election.cc:290] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b98d89b2a2444f97bab4ad389ef0c173 (127.3.243.125:39339), fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589)
03:00:20.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.602368 4096 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b98d89b2a2444f97bab4ad389ef0c173" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "28b4d256f6804d7aab5e92586dc1b038" is_pre_election: true
03:00:20.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.602536 4178 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "28b4d256f6804d7aab5e92586dc1b038" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b98d89b2a2444f97bab4ad389ef0c173" is_pre_election: true
03:00:20.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.602598 4096 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b98d89b2a2444f97bab4ad389ef0c173 in term 2.
03:00:20.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.602746 4178 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 28b4d256f6804d7aab5e92586dc1b038 in term 2.
03:00:20.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.603192 4071 leader_election.cc:304] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 28b4d256f6804d7aab5e92586dc1b038, b98d89b2a2444f97bab4ad389ef0c173; no voters:
03:00:20.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.603494 4073 leader_election.cc:336] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.603514 4857 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 2 FOLLOWER]: Leader pre-election won for term 3
03:00:20.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.603641 4857 raft_consensus.cc:493] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:00:20.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.603739 4857 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 2 FOLLOWER]: Advancing to term 3
03:00:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.603797 4154 leader_election.cc:304] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 28b4d256f6804d7aab5e92586dc1b038, b98d89b2a2444f97bab4ad389ef0c173; no voters:
03:00:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.604102 4154 leader_election.cc:336] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.604146 4858 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 2 FOLLOWER]: Leader pre-election won for term 3
03:00:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.604240 4858 raft_consensus.cc:493] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:00:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.604333 4858 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 2 FOLLOWER]: Advancing to term 3
03:00:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.606426 4857 raft_consensus.cc:515] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.606876 4858 raft_consensus.cc:515] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.607103 4857 leader_election.cc:290] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 3 election: Requested vote from peers b98d89b2a2444f97bab4ad389ef0c173 (127.3.243.125:39339), fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589)
03:00:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.607589 4858 leader_election.cc:290] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 3 election: Requested vote from peers 28b4d256f6804d7aab5e92586dc1b038 (127.3.243.126:33571), fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589)
03:00:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.607563 4178 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "28b4d256f6804d7aab5e92586dc1b038" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b98d89b2a2444f97bab4ad389ef0c173"
03:00:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.607841 4178 raft_consensus.cc:2393] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 3 FOLLOWER]: Leader election vote request: Denying vote to candidate 28b4d256f6804d7aab5e92586dc1b038 in current term 3: Already voted for candidate b98d89b2a2444f97bab4ad389ef0c173 in this term.
03:00:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.608224 4096 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b98d89b2a2444f97bab4ad389ef0c173" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "28b4d256f6804d7aab5e92586dc1b038"
03:00:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.608480 4096 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 3 FOLLOWER]: Leader election vote request: Denying vote to candidate b98d89b2a2444f97bab4ad389ef0c173 in current term 3: Already voted for candidate 28b4d256f6804d7aab5e92586dc1b038 in this term.
03:00:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.608879 4073 leader_election.cc:336] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.609006 4073 leader_election.cc:304] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [CANDIDATE]: Term 3 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 28b4d256f6804d7aab5e92586dc1b038; no voters: b98d89b2a2444f97bab4ad389ef0c173, fdd32a3c604d4763b45610d82e3138c4
03:00:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.609278 4857 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 3 FOLLOWER]: Leader election lost for term 3. Reason: could not achieve majority
03:00:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.609568 4154 leader_election.cc:336] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.609679 4154 leader_election.cc:304] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 3 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b98d89b2a2444f97bab4ad389ef0c173; no voters: 28b4d256f6804d7aab5e92586dc1b038, fdd32a3c604d4763b45610d82e3138c4
03:00:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.609900 4858 raft_consensus.cc:2749] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 3 FOLLOWER]: Leader election lost for term 3. Reason: could not achieve majority
03:00:20.644 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.645 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.724 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.726 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.726 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, deferred=Deferred@1375502863(state=PENDING, result=null, callback=(continuation of Deferred@235076793 after retry RPC after error@1122227778) -> (continuation of Deferred@37382476 after retry RPC after error@937652222) -> (continuation of Deferred@782084751 after retry RPC after error@1913665752) -> (continuation of Deferred@590438367 after retry RPC after error@251742424) -> (continuation of Deferred@1275810405 after retry RPC after error@1068393924) -> (continuation of Deferred@373824073 after retry RPC after error@319473616) -> (continuation of Deferred@84162478 after retry RPC after error@1229517455), errback=(continuation of Deferred@235076793 after retry RPC after error@1122227778) -> (continuation of Deferred@37382476 after retry RPC after error@937652222) -> (continuation of Deferred@782084751 after retry RPC after error@1913665752) -> (continuation of Deferred@590438367 after retry RPC after error@251742424) -> (continuation of Deferred@1275810405 after retry RPC after error@1068393924) -> (continuation of Deferred@373824073 after retry RPC after error@319473616) -> (continuation of Deferred@84162478 after retry RPC after error@1229517455)))
03:00:20.727 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [102ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [162ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, deferred=Deferred@1375502863(state=PENDING, result=null, callback=(continuation of Deferred@235076793 after retry RPC after error@1122227778) -> (continuation of Deferred@37382476 after retry RPC after error@937652222) -> (continuation of Deferred@782084751 after retry RPC after error@1913665752) -> (continuation of Deferred@590438367 after retry RPC after error@251742424) -> (continuation of Deferred@1275810405 after retry RPC after error@1068393924) -> (continuation of Deferred@373824073 after retry RPC after error@319473616) -> (continuation of Deferred@84162478 after retry RPC after error@1229517455), errback=(continuation of Deferred@235076793 after retry RPC after error@1122227778) -> (continuation of Deferred@37382476 after retry RPC after error@937652222) -> (continuation of Deferred@782084751 after retry RPC after error@1913665752) -> (continuation of Deferred@590438367 after retry RPC after error@251742424) -> (continuation of Deferred@1275810405 after retry RPC after error@1068393924) -> (continuation of Deferred@373824073 after retry RPC after error@319473616) -> (continuation of Deferred@84162478 after retry RPC after error@1229517455)))
03:00:20.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.736948 4639 heartbeater.cc:646] Failed to heartbeat to 127.3.243.124:41589 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.744 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.746 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.764 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.765 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.766 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.784 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.804 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.805 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.805 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.805 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.806 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.824 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.825 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.825 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.844 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.846 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.847 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.870994 4665 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.133s from last keepalive heartbeat (effective timeout is 1.000s)
03:00:20.924 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:20.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.925 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.925 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:20.926 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589; will retry
03:00:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.944391 4858 raft_consensus.cc:493] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:00:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.944602 4858 raft_consensus.cc:515] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.945338 4858 leader_election.cc:290] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 28b4d256f6804d7aab5e92586dc1b038 (127.3.243.126:33571), fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589)
03:00:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.945983 4096 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b98d89b2a2444f97bab4ad389ef0c173" candidate_term: 4 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "28b4d256f6804d7aab5e92586dc1b038" is_pre_election: true
03:00:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.946254 4096 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b98d89b2a2444f97bab4ad389ef0c173 in term 3.
03:00:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.946782 4154 leader_election.cc:304] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 28b4d256f6804d7aab5e92586dc1b038, b98d89b2a2444f97bab4ad389ef0c173; no voters:
03:00:20.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.947058 4154 leader_election.cc:336] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.947054 4858 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 3 FOLLOWER]: Leader pre-election won for term 4
03:00:20.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.947211 4858 raft_consensus.cc:493] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:00:20.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.947291 4858 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 3 FOLLOWER]: Advancing to term 4
03:00:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.949139 4858 raft_consensus.cc:515] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.949618 4858 leader_election.cc:290] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 4 election: Requested vote from peers 28b4d256f6804d7aab5e92586dc1b038 (127.3.243.126:33571), fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589)
03:00:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.950078 4096 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b98d89b2a2444f97bab4ad389ef0c173" candidate_term: 4 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "28b4d256f6804d7aab5e92586dc1b038"
03:00:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.950285 4096 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 3 FOLLOWER]: Advancing to term 4
03:00:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:20.950850 4154 leader_election.cc:336] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111)
03:00:20.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.952365 4096 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b98d89b2a2444f97bab4ad389ef0c173 in term 4.
03:00:20.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.952802 4154 leader_election.cc:304] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 28b4d256f6804d7aab5e92586dc1b038, b98d89b2a2444f97bab4ad389ef0c173; no voters: fdd32a3c604d4763b45610d82e3138c4
03:00:20.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.953069 4858 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 4 FOLLOWER]: Leader election won for term 4
03:00:20.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.953473 4858 raft_consensus.cc:697] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [term 4 LEADER]: Becoming Leader. State: Replica: b98d89b2a2444f97bab4ad389ef0c173, State: Running, Role: LEADER
03:00:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.953922 4858 consensus_queue.cc:237] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } }
03:00:20.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.955459 4877 sys_catalog.cc:455] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b98d89b2a2444f97bab4ad389ef0c173. Latest consensus state: current_term: 4 leader_uuid: "b98d89b2a2444f97bab4ad389ef0c173" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:20.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.955732 4877 sys_catalog.cc:458] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: This master's current role is: LEADER
03:00:20.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.956367 4879 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:00:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.958766 4879 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=6617aa2658c441eeaf45d6297335615b]
03:00:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.959273 4879 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=6be52772103744a9931a2c9fb2f6c884]
03:00:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.960954 4879 tablet_loader.cc:96] loaded metadata for tablet c44e76cefd944f01b843f78eb75481d4 (table txn_manager_fallback_rolling [id=6be52772103744a9931a2c9fb2f6c884])
03:00:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.961324 4879 tablet_loader.cc:96] loaded metadata for tablet e4dcf6f0fca540eeb668100804b36df1 (table txn_manager_fallback_rolling [id=6be52772103744a9931a2c9fb2f6c884])
03:00:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.961649 4879 tablet_loader.cc:96] loaded metadata for tablet fc3ecc41943943d9ad97893bab2956ef (table kudu_system.kudu_transactions [id=6617aa2658c441eeaf45d6297335615b])
03:00:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.961884 4879 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:00:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.962706 4879 catalog_manager.cc:1269] Loaded cluster ID: cf189fddef194b5b91c14a3d072e7bc4
03:00:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.962796 4879 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:00:20.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.964058 4879 catalog_manager.cc:1514] Loading token signing keys...
03:00:20.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.964987 4879 catalog_manager.cc:6038] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173: Loaded TSK: 0
03:00:20.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:20.965817 4879 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:00:21.004 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [22ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [103ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [182ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.3.243.125:39339,127.3.243.126:33571,127.3.243.124:41589) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, deferred=Deferred@720308246(state=PENDING, result=null, callback=(continuation of Deferred@655925470 after retry RPC after error@172380021) -> (continuation of Deferred@253432282 after retry RPC after error@1566967703) -> (continuation of Deferred@1058689717 after retry RPC after error@127901636) -> (continuation of Deferred@1356712748 after retry RPC after error@1030370335) -> (continuation of Deferred@1351108349 after retry RPC after error@722637463) -> (continuation of Deferred@363374706 after retry RPC after error@2031008788) -> (continuation of Deferred@1805890150 after retry RPC after error@1410991023), errback=(continuation of Deferred@655925470 after retry RPC after error@172380021) -> (continuation of Deferred@253432282 after retry RPC after error@1566967703) -> (continuation of Deferred@1058689717 after retry RPC after error@127901636) -> (continuation of Deferred@1356712748 after retry RPC after error@1030370335) -> (continuation of Deferred@1351108349 after retry RPC after error@722637463) -> (continuation of Deferred@363374706 after retry RPC after error@2031008788) -> (continuation of Deferred@1805890150 after retry RPC after error@1410991023)))
03:00:21.007 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
03:00:21.025 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.3.243.124:41589(127.3.243.124:41589)] connecting to peer
03:00:21.026 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:21.026 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.124:41589(127.3.243.124:41589)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:21.026 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.243.124:41589: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589
03:00:21.028 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:00:21.028 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=4), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.125:39339, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.126:33571, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.243.124:41589, [2ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.126:33571: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.124:41589: Network error: Failed to connect to peer master-127.3.243.124:41589(127.3.243.124:41589): Connection refused: /127.3.243.124:41589, [4ms] Sub RPC ConnectToMaster: received response from server master-127.3.243.125:39339: OK, deferred=null) after lookup
03:00:21.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:21.045672 4096 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [term 4 FOLLOWER]: Refusing update from remote peer b98d89b2a2444f97bab4ad389ef0c173: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 4 index: 12. (index mismatch)
03:00:21.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:21.046505 4877 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [LEADER]: Connected to new peer: Peer: permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
03:00:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:21.048420 4857 sys_catalog.cc:455] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b98d89b2a2444f97bab4ad389ef0c173. Latest consensus state: current_term: 4 leader_uuid: "b98d89b2a2444f97bab4ad389ef0c173" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:21.048661 4857 sys_catalog.cc:458] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [sys.catalog]: This master's current role is: FOLLOWER
03:00:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:21.071367 4154 consensus_peers.cc:597] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 -> Peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Couldn't send request to peer fdd32a3c604d4763b45610d82e3138c4. Status: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:00:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:21.071275 4858 sys_catalog.cc:455] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "b98d89b2a2444f97bab4ad389ef0c173" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:21.071660 4858 sys_catalog.cc:458] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: This master's current role is: LEADER
03:00:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:21.072475 4857 sys_catalog.cc:455] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "b98d89b2a2444f97bab4ad389ef0c173" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:21.072686 4857 sys_catalog.cc:458] T 00000000000000000000000000000000 P 28b4d256f6804d7aab5e92586dc1b038 [sys.catalog]: This master's current role is: FOLLOWER
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:21.092522 4168 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56050:
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:00:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:00:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:00:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:00:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:00:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:00:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:21.097821 4168 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:00:21.115 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 04ef8b302cb34e32beaf73bdb49909ec (127.3.243.67:43563): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1765335619 last_transition_timestamp: 1765335620
03:00:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:21.562778 4154 consensus_peers.cc:597] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 -> Peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Couldn't send request to peer fdd32a3c604d4763b45610d82e3138c4. Status: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:00:21.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:21.687076 4154 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111) [suppressed 16 similar messages]
03:00:21.890 [DEBUG - main] (Connection.java:575) [peer master-127.3.243.126:33571(127.3.243.126:33571)] disconnecting while in state READY
03:00:21.891 [DEBUG - main] (Connection.java:575) [peer master-127.3.243.124:41589(127.3.243.124:41589)] disconnecting while in state TERMINATED
03:00:21.891 [DEBUG - main] (Connection.java:575) [peer master-127.3.243.125:39339(127.3.243.125:39339)] disconnecting while in state READY
03:00:21.891 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.3.243.126:33571(127.3.243.126:33571)] handling channelInactive
03:00:21.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.243.126:33571(127.3.243.126:33571)] cleaning up while in state READY due to: connection closed
03:00:21.892 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:00:21.893 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.243.125:39339(127.3.243.125:39339)] handling channelInactive
03:00:21.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.243.125:39339(127.3.243.125:39339)] cleaning up while in state READY due to: connection closed
03:00:21.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:21.893362 4045 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task7KZUIc/build/asan/bin/kudu pid 4339
03:00:21.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:21.906296 4492 generic_service.cc:196] Checking for leaks (request via RPC)
03:00:22.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.006961 4499 heartbeater.cc:499] Master 127.3.243.125:39339 was elected leader, sending a full tablet report...
03:00:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.008522 4045 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task7KZUIc/build/asan/bin/kudu with pid 4339
03:00:22.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.023820 4045 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task7KZUIc/build/asan/bin/kudu pid 4502
03:00:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.030443 4634 generic_service.cc:196] Checking for leaks (request via RPC)
03:00:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.093762 4782 heartbeater.cc:499] Master 127.3.243.125:39339 was elected leader, sending a full tablet report...
03:00:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.100296 4056 master_runner.cc:186] Error getting master registration for 127.3.243.124:41589: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111),
03:00:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.100556 4056 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:00:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:22.109262 4660 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.243.65:36037: connect: Connection refused (error 111)
03:00:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:22.110816 4660 consensus_peers.cc:597] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec -> Peer bcf4e423f7b845e3ae355799a953cfa2 (127.3.243.65:36037): Couldn't send request to peer bcf4e423f7b845e3ae355799a953cfa2. Status: Network error: Client connection negotiation failed: client connection to 127.3.243.65:36037: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:00:22.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:22.129782 4154 consensus_peers.cc:597] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 -> Peer fdd32a3c604d4763b45610d82e3138c4 (127.3.243.124:41589): Couldn't send request to peer fdd32a3c604d4763b45610d82e3138c4. Status: Network error: Client connection negotiation failed: client connection to 127.3.243.124:41589: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
03:00:22.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.131340 4045 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task7KZUIc/build/asan/bin/kudu with pid 4502
03:00:22.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:22.146488 4660 connection.cc:537] client connection to 127.3.243.66:43141 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:00:22.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.147153 4045 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task7KZUIc/build/asan/bin/kudu pid 4644
03:00:22.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:22.147855 4660 consensus_peers.cc:597] T c44e76cefd944f01b843f78eb75481d4 P 04ef8b302cb34e32beaf73bdb49909ec -> Peer 2bd9c23e31ad450f8cf481e0dfdf5f0e (127.3.243.66:43141): Couldn't send request to peer 2bd9c23e31ad450f8cf481e0dfdf5f0e. Status: Network error: Client connection negotiation failed: client connection to 127.3.243.66:43141: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:00:22.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.154023 4775 generic_service.cc:196] Checking for leaks (request via RPC)
03:00:22.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.277037 4045 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task7KZUIc/build/asan/bin/kudu with pid 4644
03:00:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.295509 4045 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task7KZUIc/build/asan/bin/kudu pid 4056
03:00:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.296665 4127 generic_service.cc:196] Checking for leaks (request via RPC)
03:00:22.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.409657 4045 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task7KZUIc/build/asan/bin/kudu with pid 4056
03:00:22.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:22.434749 4154 connection.cc:537] client connection to 127.3.243.126:33571 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:00:22.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.435087 4045 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task7KZUIc/build/asan/bin/kudu pid 4134
03:00:22.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251210 03:00:22.436373 4154 consensus_peers.cc:597] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 -> Peer 28b4d256f6804d7aab5e92586dc1b038 (127.3.243.126:33571): Couldn't send request to peer 28b4d256f6804d7aab5e92586dc1b038. Status: Network error: Client connection negotiation failed: client connection to 127.3.243.126:33571: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:00:22.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.436679 4208 generic_service.cc:196] Checking for leaks (request via RPC)
03:00:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.559610 4877 sys_catalog.cc:455] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "b98d89b2a2444f97bab4ad389ef0c173" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "28b4d256f6804d7aab5e92586dc1b038" member_type: VOTER last_known_addr { host: "127.3.243.126" port: 33571 } } peers { permanent_uuid: "b98d89b2a2444f97bab4ad389ef0c173" member_type: VOTER last_known_addr { host: "127.3.243.125" port: 39339 } } peers { permanent_uuid: "fdd32a3c604d4763b45610d82e3138c4" member_type: VOTER last_known_addr { host: "127.3.243.124" port: 41589 } } }
03:00:22.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.559918 4877 sys_catalog.cc:458] T 00000000000000000000000000000000 P b98d89b2a2444f97bab4ad389ef0c173 [sys.catalog]: This master's current role is: LEADER
03:00:22.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251210 03:00:22.565248 4045 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task7KZUIc/build/asan/bin/kudu with pid 4134
03:00:22.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-10T03:00:22Z chronyd exiting
03:00:22.719 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)