08:03:48.732 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:03:48.732 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskWhNLRi/test-tmp
08:03:48.732 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
08:03:48.732 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskWhNLRi/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
08:03:48.733 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:03:49.016 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:49.016 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:03:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.018491 802 mini_oidc.cc:150] Starting JWKS server
08:03:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.020467 802 webserver.cc:492] Webserver started at http://127.0.0.1:43367/ using document root <none> and password file <none>
08:03:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.020661 802 mini_oidc.cc:186] Starting OIDC Discovery server
08:03:49.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.021065 802 webserver.cc:492] Webserver started at http://127.0.0.1:46341/ using document root <none> and password file <none>
08:03:49.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-25T08:03:49Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:03:49.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-25T08:03:49Z Disabled control of system clock
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.045387 802 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/wal
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/data
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/logs
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/data/info.pb
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.190:34159
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.190
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43367/jwks.json
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:49.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.319221 813 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:49.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.319456 813 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:49.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.319535 813 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:49.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.326058 813 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:49.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.326122 813 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:49.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.326164 813 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:49.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.326197 813 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:49.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.326249 813 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:49.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.326282 813 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:49.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.326328 813 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43367/jwks.json
08:03:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.334774 813 master_runner.cc:386] Master server non-default flags:
08:03:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/data
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/wal
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.190:34159
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43367/jwks.json
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/data/info.pb
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.190
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/logs
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 07:43:22 UTC on 24a791456cd2
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8708
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:49.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.336009 813 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:49.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.337424 813 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.344656 819 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.344708 818 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.345263 821 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.345683 813 server_base.cc:1047] running on GCE node
08:03:49.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.346259 813 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:49.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.347328 813 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:49.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.348529 813 hybrid_clock.cc:648] HybridClock initialized: now 1761379429348461 us; error 42 us; skew 500 ppm
08:03:49.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.348841 813 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:49.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.350720 813 webserver.cc:492] Webserver started at http://127.0.200.190:45049/ using document root <none> and password file <none>
08:03:49.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.351315 813 fs_manager.cc:362] Metadata directory not provided
08:03:49.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.351440 813 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:49.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.351763 813 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.353698 813 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/data/instance:
08:03:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ed5e67dc759449b960c2c71ee039c42"
08:03:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:49 on dist-test-slave-37ck"
08:03:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.354336 813 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/wal/instance:
08:03:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ed5e67dc759449b960c2c71ee039c42"
08:03:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:49 on dist-test-slave-37ck"
08:03:49.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.358776 813 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
08:03:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.361670 827 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:49.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.362658 813 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
08:03:49.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.362797 813 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/data,/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/wal
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ed5e67dc759449b960c2c71ee039c42"
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:49 on dist-test-slave-37ck"
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.362950 813 fs_report.cc:389] FS layout report
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/wal
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/wal
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/data/data
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:49.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.374629 813 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:49.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.375281 813 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:49.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.375610 813 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.395362 813 rpc_server.cc:307] RPC server started. Bound to: 127.0.200.190:34159
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.395407 888 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.200.190:34159 every 8 connection(s)
08:03:49.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.396590 813 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/data/info.pb
08:03:49.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.397799 802 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu as pid 813
08:03:49.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.398058 802 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-0/wal/instance
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.400120 802 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/wal
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/data
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/logs
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/data/info.pb
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.189:44271
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.189
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43367/jwks.json
08:03:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.400403 889 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:49.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.407394 889 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } has no permanent_uuid. Determining permanent_uuid...
08:03:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.422482 889 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } has no permanent_uuid. Determining permanent_uuid...
08:03:49.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.423560 830 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.200.189:44271: connect: Connection refused (error 111)
08:03:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.424917 889 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.189:44271: Network error: Client connection negotiation failed: client connection to 127.0.200.189:44271: connect: Connection refused (error 111)
08:03:49.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.474243 889 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } attempt: 1
08:03:49.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.475807 889 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.189:44271: Network error: Client connection negotiation failed: client connection to 127.0.200.189:44271: connect: Connection refused (error 111)
08:03:49.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.544030 889 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } attempt: 2
08:03:49.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.545687 889 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.189:44271: Network error: Client connection negotiation failed: client connection to 127.0.200.189:44271: connect: Connection refused (error 111)
08:03:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.636951 889 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } attempt: 3
08:03:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.638633 889 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.189:44271: Network error: Client connection negotiation failed: client connection to 127.0.200.189:44271: connect: Connection refused (error 111)
08:03:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.686707 891 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.686937 891 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.687008 891 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:49.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.693989 891 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:49.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.694057 891 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:49.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.694101 891 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:49.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.694156 891 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:49.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.694209 891 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:49.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.694245 891 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:49.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.694288 891 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43367/jwks.json
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.703289 891 master_runner.cc:386] Master server non-default flags:
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/data
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/wal
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.189:44271
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43367/jwks.json
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/data/info.pb
08:03:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.189
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/logs
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 07:43:22 UTC on 24a791456cd2
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8708
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.704566 891 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:49.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.706148 891 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.713248 900 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.713281 899 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.713824 902 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.714699 891 server_base.cc:1047] running on GCE node
08:03:49.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.715337 891 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:49.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.716431 891 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:49.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.717646 891 hybrid_clock.cc:648] HybridClock initialized: now 1761379429717594 us; error 36 us; skew 500 ppm
08:03:49.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.717887 891 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.719794 891 webserver.cc:492] Webserver started at http://127.0.200.189:32773/ using document root <none> and password file <none>
08:03:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.720364 891 fs_manager.cc:362] Metadata directory not provided
08:03:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.720453 891 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.720700 891 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.722561 891 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/data/instance:
08:03:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9809071a62ca479b8178425bef946094"
08:03:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:49 on dist-test-slave-37ck"
08:03:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.723152 891 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/wal/instance:
08:03:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9809071a62ca479b8178425bef946094"
08:03:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:49 on dist-test-slave-37ck"
08:03:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.727324 891 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.007s sys 0.000s
08:03:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.730141 908 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.731142 891 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.731254 891 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/data,/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/wal
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9809071a62ca479b8178425bef946094"
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:49 on dist-test-slave-37ck"
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.731376 891 fs_report.cc:389] FS layout report
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/wal
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/wal
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/data/data
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:49.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.760277 891 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:49.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.760906 891 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:49.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.761128 891 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:49.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.781456 891 rpc_server.cc:307] RPC server started. Bound to: 127.0.200.189:44271
08:03:49.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.781491 969 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.200.189:44271 every 8 connection(s)
08:03:49.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.781893 889 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } attempt: 4
08:03:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.782943 891 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/data/info.pb
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.786365 802 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu as pid 891
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.786518 802 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-1/wal/instance
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.786980 970 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.787649 802 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/logs
08:03:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data/info.pb
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.188:39611
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.188
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43367/jwks.json
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:49.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.793433 970 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } has no permanent_uuid. Determining permanent_uuid...
08:03:49.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.799011 889 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } has no permanent_uuid. Determining permanent_uuid...
08:03:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.801080 889 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.188:39611: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.801961 970 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } has no permanent_uuid. Determining permanent_uuid...
08:03:49.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.806672 970 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } has no permanent_uuid. Determining permanent_uuid...
08:03:49.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.807394 911 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:49.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.808739 970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.188:39611: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:49.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.858093 970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } attempt: 1
08:03:49.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.859786 970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.188:39611: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.860271 889 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } attempt: 1
08:03:49.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.861881 889 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.188:39611: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:49.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.927991 970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } attempt: 2
08:03:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:49.929076 889 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } attempt: 2
08:03:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.930042 970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.188:39611: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:49.930673 889 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.188:39611: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:50.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.021265 970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } attempt: 3
08:03:50.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.023064 970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.188:39611: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:50.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.030843 889 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } attempt: 3
08:03:50.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.032266 889 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.188:39611: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.075878 973 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.076112 973 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.076182 973 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:50.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.083149 973 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:50.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.083216 973 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:50.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.083266 973 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:50.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.083304 973 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:50.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.083353 973 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:50.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.083390 973 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:50.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.083433 973 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43367/jwks.json
08:03:50.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.091796 973 master_runner.cc:386] Master server non-default flags:
08:03:50.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.188:39611
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43367/jwks.json
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data/info.pb
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.188
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/logs
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 07:43:22 UTC on 24a791456cd2
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8708
08:03:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:50.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.093027 973 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:50.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.094503 973 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.101626 981 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.101639 980 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.102399 983 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.102808 973 server_base.cc:1047] running on GCE node
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.103587 973 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.104740 973 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.106086 973 hybrid_clock.cc:648] HybridClock initialized: now 1761379430106009 us; error 31 us; skew 500 ppm
08:03:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.106415 973 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.108438 973 webserver.cc:492] Webserver started at http://127.0.200.188:35305/ using document root <none> and password file <none>
08:03:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.109117 973 fs_manager.cc:362] Metadata directory not provided
08:03:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.109251 973 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.109567 973 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:50.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.111732 973 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data/instance:
08:03:50.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c116f66397249a491b1d9259e219594"
08:03:50.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:50 on dist-test-slave-37ck"
08:03:50.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.112507 973 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal/instance:
08:03:50.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c116f66397249a491b1d9259e219594"
08:03:50.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:50 on dist-test-slave-37ck"
08:03:50.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.116930 973 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
08:03:50.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.119872 989 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.120994 973 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
08:03:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.121150 973 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data,/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal
08:03:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c116f66397249a491b1d9259e219594"
08:03:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:50 on dist-test-slave-37ck"
08:03:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.121321 973 fs_report.cc:389] FS layout report
08:03:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal
08:03:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal
08:03:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data/data
08:03:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:50.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.150259 973 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:50.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.150965 973 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:50.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.151248 973 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:50.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.166245 970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } attempt: 4
08:03:50.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.168171 970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.200.188:39611: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.173048 973 rpc_server.cc:307] RPC server started. Bound to: 127.0.200.188:39611
08:03:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.173086 1050 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.200.188:39611 every 8 connection(s)
08:03:50.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.174364 973 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data/info.pb
08:03:50.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.177771 1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:50.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.182819 802 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu as pid 973
08:03:50.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.182997 802 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal/instance
08:03:50.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.182806 1051 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } has no permanent_uuid. Determining permanent_uuid...
08:03:50.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.195542 1051 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } has no permanent_uuid. Determining permanent_uuid...
08:03:50.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.202294 1051 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } has no permanent_uuid. Determining permanent_uuid...
08:03:50.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.202473 889 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } attempt: 4
08:03:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.217384 889 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42: Bootstrap starting.
08:03:50.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.217877 1051 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: Bootstrap starting.
08:03:50.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.221843 1051 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: Neither blocks nor log segments found. Creating new log.
08:03:50.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.221875 889 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42: Neither blocks nor log segments found. Creating new log.
08:03:50.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.223106 1051 log.cc:826] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: Log is configured to *not* fsync() on all Append() calls
08:03:50.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.223282 889 log.cc:826] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42: Log is configured to *not* fsync() on all Append() calls
08:03:50.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.225392 889 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42: No bootstrap required, opened a new log
08:03:50.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.225468 1051 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: No bootstrap required, opened a new log
08:03:50.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.230935 1051 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:50.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.231073 889 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:50.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.231247 1051 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:50.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.231310 889 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.231395 889 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9ed5e67dc759449b960c2c71ee039c42, State: Initialized, Role: FOLLOWER
08:03:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.231395 1051 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4c116f66397249a491b1d9259e219594, State: Initialized, Role: FOLLOWER
08:03:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.232007 889 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.232280 1051 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.232646 1062 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.232873 1062 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.232996 1063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.233268 1063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [sys.catalog]: This master's current role is: FOLLOWER
08:03:50.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.234151 889 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [sys.catalog]: configured and running, proceeding with master startup.
08:03:50.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.234586 1062 leader_election.cc:290] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159), 9809071a62ca479b8178425bef946094 (127.0.200.189:44271)
08:03:50.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.234890 1062 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:50.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.235090 1062 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: This master's current role is: FOLLOWER
08:03:50.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.236111 854 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c116f66397249a491b1d9259e219594" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9ed5e67dc759449b960c2c71ee039c42" is_pre_election: true
08:03:50.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.236672 854 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4c116f66397249a491b1d9259e219594 in term 0.
08:03:50.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.237465 993 leader_election.cc:304] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4c116f66397249a491b1d9259e219594, 9ed5e67dc759449b960c2c71ee039c42; no voters:
08:03:50.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.237875 1062 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:50.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.238063 1062 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:50.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.238165 1062 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 0 FOLLOWER]: Advancing to term 1
08:03:50.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.241739 1051 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: configured and running, proceeding with master startup.
08:03:50.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.242071 1062 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.243117 1062 leader_election.cc:290] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [CANDIDATE]: Term 1 election: Requested vote from peers 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159), 9809071a62ca479b8178425bef946094 (127.0.200.189:44271)
08:03:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.243638 854 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c116f66397249a491b1d9259e219594" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9ed5e67dc759449b960c2c71ee039c42"
08:03:50.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.243894 854 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 0 FOLLOWER]: Advancing to term 1
08:03:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.247761 854 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4c116f66397249a491b1d9259e219594 in term 1.
08:03:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.248538 993 leader_election.cc:304] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4c116f66397249a491b1d9259e219594, 9ed5e67dc759449b960c2c71ee039c42; no voters:
08:03:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.248898 1062 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 1 FOLLOWER]: Leader election won for term 1
08:03:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.249078 1062 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 1 LEADER]: Becoming Leader. State: Replica: 4c116f66397249a491b1d9259e219594, State: Running, Role: LEADER
08:03:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.249626 1062 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.253634 1064 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4c116f66397249a491b1d9259e219594. Latest consensus state: current_term: 1 leader_uuid: "4c116f66397249a491b1d9259e219594" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.253966 1064 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: This master's current role is: LEADER
08:03:50.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.263962 1074 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:03:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.281419 1083 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:03:50.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.282414 1074 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:03:50.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.285815 813 master_runner.cc:186] Error getting master registration for 127.0.200.189:44271: OK, instance_id { permanent_uuid: "9809071a62ca479b8178425bef946094" instance_seqno: 1761379429776555 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:50.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.297676 813 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.303881 853 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 1 FOLLOWER]: Refusing update from remote peer 4c116f66397249a491b1d9259e219594: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.304666 1062 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.305477 973 master_runner.cc:186] Error getting master registration for 127.0.200.189:44271: OK, instance_id { permanent_uuid: "9809071a62ca479b8178425bef946094" instance_seqno: 1761379429776555 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.311152 973 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.314538 1063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4c116f66397249a491b1d9259e219594. Latest consensus state: current_term: 1 leader_uuid: "4c116f66397249a491b1d9259e219594" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.314755 1063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [sys.catalog]: This master's current role is: FOLLOWER
08:03:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.317824 1086 mvcc.cc:204] Tried to move back new op lower bound from 7214610146512039936 to 7214610146307944448. Current Snapshot: MvccSnapshot[applied={T|T < 7214610146512039936}]
08:03:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.318084 1091 mvcc.cc:204] Tried to move back new op lower bound from 7214610146512039936 to 7214610146307944448. Current Snapshot: MvccSnapshot[applied={T|T < 7214610146512039936}]
08:03:50.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.323741 1063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4c116f66397249a491b1d9259e219594" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:50.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.323913 1063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [sys.catalog]: This master's current role is: FOLLOWER
08:03:50.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.329249 1064 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4c116f66397249a491b1d9259e219594" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:50.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.329427 1064 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: This master's current role is: LEADER
08:03:50.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.329861 1074 catalog_manager.cc:1357] Generated new cluster ID: 11103c47897d43f1bc346107e55096ee
08:03:50.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.329954 1074 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:03:50.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.347016 1074 catalog_manager.cc:1380] Generated new certificate authority record
08:03:50.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.349236 1074 catalog_manager.cc:1514] Loading token signing keys...
08:03:50.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.374239 1074 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: Generated new TSK 0
08:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.375325 1074 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.383996 802 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/wal
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/data
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/logs
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/data/info.pb
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.129:0
08:03:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.200.129
08:03:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.129
08:03:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:50.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.467387 970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } attempt: 5
08:03:50.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.479231 970 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094: Bootstrap starting.
08:03:50.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.482873 970 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094: Neither blocks nor log segments found. Creating new log.
08:03:50.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.484165 970 log.cc:826] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094: Log is configured to *not* fsync() on all Append() calls
08:03:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.492841 970 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094: No bootstrap required, opened a new log
08:03:50.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.500188 970 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:50.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.500923 970 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:50.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.501112 970 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9809071a62ca479b8178425bef946094, State: Initialized, Role: FOLLOWER
08:03:50.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.501784 970 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:50.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.503185 1097 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:50.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.503724 1097 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: This master's current role is: FOLLOWER
08:03:50.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.504127 970 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: configured and running, proceeding with master startup.
08:03:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.504423 933 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 0 FOLLOWER]: Advancing to term 1
08:03:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.507647 935 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c116f66397249a491b1d9259e219594" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9809071a62ca479b8178425bef946094" is_pre_election: true
08:03:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.508351 935 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:57718 (request call id 1) took 273 ms (client timeout 303 ms). Trace:
08:03:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.508499 935 rpcz_store.cc:269] 1025 08:03:50.234675 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:03:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:50.237068 (+ 2393us) service_pool.cc:225] Handling call
08:03:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:50.508294 (+271226us) inbound_call.cc:173] Queueing success response
08:03:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
08:03:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.510124 934 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c116f66397249a491b1d9259e219594" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9809071a62ca479b8178425bef946094"
08:03:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.510442 934 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:57718 (request call id 2) took 267 ms (client timeout 329 ms). Trace:
08:03:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.510536 934 rpcz_store.cc:269] 1025 08:03:50.243248 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:03:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:50.243459 (+ 211us) service_pool.cc:225] Handling call
08:03:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:50.510409 (+266950us) inbound_call.cc:173] Queueing success response
08:03:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
08:03:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.511992 933 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 1 FOLLOWER]: Refusing update from remote peer 4c116f66397249a491b1d9259e219594: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.512699 1062 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.513401 914 tablet.cc:2378] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:50.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.557327 1097 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4c116f66397249a491b1d9259e219594. Latest consensus state: current_term: 1 leader_uuid: "4c116f66397249a491b1d9259e219594" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:50.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.557551 1097 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: This master's current role is: FOLLOWER
08:03:50.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.558210 1105 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4c116f66397249a491b1d9259e219594" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:50.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.558669 1105 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: This master's current role is: FOLLOWER
08:03:50.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.581964 1113 catalog_manager.cc:1269] Loaded cluster ID: 11103c47897d43f1bc346107e55096ee
08:03:50.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.582201 1113 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094: loading cluster ID for follower catalog manager: success
08:03:50.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.585795 1113 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094: acquiring CA information for follower catalog manager: success
08:03:50.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.588160 1113 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.613993 1062 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4c116f66397249a491b1d9259e219594" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.614265 1062 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: This master's current role is: LEADER
08:03:50.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.779019 1095 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:50.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.779379 1095 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:50.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.779565 1095 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:50.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.791039 1095 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:50.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.791177 1095 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:50.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.791235 1095 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:50.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.791280 1095 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:50.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.791384 1095 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.200.129
08:03:50.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.805763 1095 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/data
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/wal
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.129:0
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/data/info.pb
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.129
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.200.129
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/logs
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 07:43:22 UTC on 24a791456cd2
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8708
08:03:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.807526 1095 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.809713 1095 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.819864 1119 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.819828 1120 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:50.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:50.821241 1122 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:50.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.821838 1095 server_base.cc:1047] running on GCE node
08:03:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.822296 1095 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.823069 1095 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.824282 1095 hybrid_clock.cc:648] HybridClock initialized: now 1761379430824231 us; error 29 us; skew 500 ppm
08:03:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.824551 1095 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:50.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.828445 1095 webserver.cc:492] Webserver started at http://127.0.200.129:36863/ using document root <none> and password file <none>
08:03:50.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.829097 1095 fs_manager.cc:362] Metadata directory not provided
08:03:50.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.829231 1095 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:50.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.829540 1095 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:50.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.831861 1095 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/data/instance:
08:03:50.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83bed13f5bbd49e897e7ee2c1682840c"
08:03:50.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:50 on dist-test-slave-37ck"
08:03:50.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.832638 1095 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/wal/instance:
08:03:50.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83bed13f5bbd49e897e7ee2c1682840c"
08:03:50.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:50 on dist-test-slave-37ck"
08:03:50.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.837585 1095 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.003s sys 0.002s
08:03:50.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.840884 1128 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.842150 1095 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
08:03:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.842298 1095 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/data,/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/wal
08:03:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83bed13f5bbd49e897e7ee2c1682840c"
08:03:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:50 on dist-test-slave-37ck"
08:03:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.842510 1095 fs_report.cc:389] FS layout report
08:03:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/wal
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/wal
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/data/data
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:50.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.861112 1095 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:50.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.861781 1095 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:50.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.862056 1095 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:50.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.865336 1095 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:50.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.865607 1095 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
08:03:50.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.865830 1095 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:50.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.866010 1095 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.968076 1095 rpc_server.cc:307] RPC server started. Bound to: 127.0.200.129:39163
08:03:50.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.968150 1251 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.200.129:39163 every 8 connection(s)
08:03:50.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.969635 1095 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/data/info.pb
08:03:50.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.973019 802 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu as pid 1095
08:03:50.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.973220 802 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-0/wal/instance
08:03:50.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.975179 1253 heartbeater.cc:344] Connected to a master server at 127.0.200.190:34159
08:03:50.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.975636 1253 heartbeater.cc:461] Registering TS with master...
08:03:50.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.976895 1253 heartbeater.cc:507] Master 127.0.200.190:34159 requested a full tablet report, sending...
08:03:50.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.979761 844 ts_manager.cc:194] Registered new tserver with Master: 83bed13f5bbd49e897e7ee2c1682840c (127.0.200.129:39163)
08:03:50.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.981812 802 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:50.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:50.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/wal
08:03:50.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/data
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/logs
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/data/info.pb
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.130:0
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.200.130
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.130
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:50.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.994100 1254 heartbeater.cc:344] Connected to a master server at 127.0.200.189:44271
08:03:50.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.994181 1252 heartbeater.cc:344] Connected to a master server at 127.0.200.188:39611
08:03:50.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.994282 1254 heartbeater.cc:461] Registering TS with master...
08:03:50.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.994285 1252 heartbeater.cc:461] Registering TS with master...
08:03:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.994921 1252 heartbeater.cc:507] Master 127.0.200.188:39611 requested a full tablet report, sending...
08:03:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.994956 1254 heartbeater.cc:507] Master 127.0.200.189:44271 requested a full tablet report, sending...
08:03:51.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.996868 1006 ts_manager.cc:194] Registered new tserver with Master: 83bed13f5bbd49e897e7ee2c1682840c (127.0.200.129:39163)
08:03:51.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:50.998900 1006 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.0.200.129:60499
08:03:51.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.000699 925 ts_manager.cc:194] Registered new tserver with Master: 83bed13f5bbd49e897e7ee2c1682840c (127.0.200.129:39163)
08:03:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.286463 1083 catalog_manager.cc:1269] Loaded cluster ID: 11103c47897d43f1bc346107e55096ee
08:03:51.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.286594 1083 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42: loading cluster ID for follower catalog manager: success
08:03:51.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.290297 1083 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42: acquiring CA information for follower catalog manager: success
08:03:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.292515 1083 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:51.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.301236 1256 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:51.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.301448 1256 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:51.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.301527 1256 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:51.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.309036 1256 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:51.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.309146 1256 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:51.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.309201 1256 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:51.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.309233 1256 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:51.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.309309 1256 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.200.130
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.317281 1256 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/data
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/wal
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.130:0
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/data/info.pb
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.130
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:51.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.200.130
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/logs
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 07:43:22 UTC on 24a791456cd2
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8708
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.318490 1256 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:51.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.320075 1256 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:51.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.328318 1262 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.331964 1263 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.333564 1256 server_base.cc:1047] running on GCE node
08:03:51.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.333797 1265 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.334327 1256 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.335150 1256 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.336423 1256 hybrid_clock.cc:648] HybridClock initialized: now 1761379431336335 us; error 72 us; skew 500 ppm
08:03:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.336673 1256 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.338575 1256 webserver.cc:492] Webserver started at http://127.0.200.130:39119/ using document root <none> and password file <none>
08:03:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.339249 1256 fs_manager.cc:362] Metadata directory not provided
08:03:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.339351 1256 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.339697 1256 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:51.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.341688 1256 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/data/instance:
08:03:51.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c8b20a175c74159881815bc68197aa4"
08:03:51.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:51 on dist-test-slave-37ck"
08:03:51.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.342458 1256 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/wal/instance:
08:03:51.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c8b20a175c74159881815bc68197aa4"
08:03:51.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:51 on dist-test-slave-37ck"
08:03:51.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.347088 1256 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
08:03:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.350194 1271 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.351430 1256 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
08:03:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.351629 1256 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/data,/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/wal
08:03:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c8b20a175c74159881815bc68197aa4"
08:03:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:51 on dist-test-slave-37ck"
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.351802 1256 fs_report.cc:389] FS layout report
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/wal
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/wal
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/data/data
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:51.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.372793 1256 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:51.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.373512 1256 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:51.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.373776 1256 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:51.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.376695 1256 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:51.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.377028 1256 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
08:03:51.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.377135 1256 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:51.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.377220 1256 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.456256 1256 rpc_server.cc:307] RPC server started. Bound to: 127.0.200.130:41469
08:03:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.456338 1392 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.200.130:41469 every 8 connection(s)
08:03:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.457397 1256 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/data/info.pb
08:03:51.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.460695 1394 heartbeater.cc:344] Connected to a master server at 127.0.200.190:34159
08:03:51.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.461179 1394 heartbeater.cc:461] Registering TS with master...
08:03:51.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.462296 1394 heartbeater.cc:507] Master 127.0.200.190:34159 requested a full tablet report, sending...
08:03:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.464129 844 ts_manager.cc:194] Registered new tserver with Master: 9c8b20a175c74159881815bc68197aa4 (127.0.200.130:41469)
08:03:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.468149 802 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu as pid 1256
08:03:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.468326 802 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-1/wal/instance
08:03:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.468463 1393 heartbeater.cc:344] Connected to a master server at 127.0.200.188:39611
08:03:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.468611 1393 heartbeater.cc:461] Registering TS with master...
08:03:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.469283 1393 heartbeater.cc:507] Master 127.0.200.188:39611 requested a full tablet report, sending...
08:03:51.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.470221 1006 ts_manager.cc:194] Registered new tserver with Master: 9c8b20a175c74159881815bc68197aa4 (127.0.200.130:41469)
08:03:51.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.471438 1006 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.0.200.130:36131
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.474149 802 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/wal
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/data
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/logs
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/data/info.pb
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.131:0
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.200.131
08:03:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.131
08:03:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:51.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.481091 1395 heartbeater.cc:344] Connected to a master server at 127.0.200.189:44271
08:03:51.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.481205 1395 heartbeater.cc:461] Registering TS with master...
08:03:51.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.481534 1395 heartbeater.cc:507] Master 127.0.200.189:44271 requested a full tablet report, sending...
08:03:51.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.482506 925 ts_manager.cc:194] Registered new tserver with Master: 9c8b20a175c74159881815bc68197aa4 (127.0.200.130:41469)
08:03:51.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.775655 1398 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:51.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.775971 1398 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:51.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.776113 1398 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.786001 1398 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.786135 1398 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.786191 1398 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.786240 1398 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.786346 1398 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.200.131
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.798313 1398 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/data
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/wal
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.131:0
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/data/info.pb
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.131
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.200.131
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/logs
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
08:03:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 07:43:22 UTC on 24a791456cd2
08:03:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8708
08:03:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.799690 1398 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:51.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.801219 1398 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.810214 1403 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.810328 1404 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.810302 1406 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.810992 1398 server_base.cc:1047] running on GCE node
08:03:51.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.811452 1398 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:51.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.812234 1398 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:51.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.813437 1398 hybrid_clock.cc:648] HybridClock initialized: now 1761379431813384 us; error 28 us; skew 500 ppm
08:03:51.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.813685 1398 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:51.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.815423 1398 webserver.cc:492] Webserver started at http://127.0.200.131:43693/ using document root <none> and password file <none>
08:03:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.816099 1398 fs_manager.cc:362] Metadata directory not provided
08:03:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.816205 1398 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.816486 1398 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.818548 1398 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/data/instance:
08:03:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c0c510dd21c40039e7d4798a2c8a019"
08:03:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:51 on dist-test-slave-37ck"
08:03:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.819285 1398 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/wal/instance:
08:03:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c0c510dd21c40039e7d4798a2c8a019"
08:03:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:51 on dist-test-slave-37ck"
08:03:51.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.823804 1398 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.003s sys 0.001s
08:03:51.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.826577 1412 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:51.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.827728 1398 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.827884 1398 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/data,/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/wal
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c0c510dd21c40039e7d4798a2c8a019"
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:51 on dist-test-slave-37ck"
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.828089 1398 fs_report.cc:389] FS layout report
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/wal
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/wal
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/data/data
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:51.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.850453 1398 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:51.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.851100 1398 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:51.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.851362 1398 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:51.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.854035 1398 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:51.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.854151 1398 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:51.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.854241 1398 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:51.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.854301 1398 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:51.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.908372 1398 rpc_server.cc:307] RPC server started. Bound to: 127.0.200.131:46651
08:03:51.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.908422 1533 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.200.131:46651 every 8 connection(s)
08:03:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.909737 1398 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/data/info.pb
08:03:51.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.911952 1535 heartbeater.cc:344] Connected to a master server at 127.0.200.190:34159
08:03:51.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.912597 802 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu as pid 1398
08:03:51.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.912740 802 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/ts-2/wal/instance
08:03:51.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.916930 1535 heartbeater.cc:461] Registering TS with master...
08:03:51.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.917918 1535 heartbeater.cc:507] Master 127.0.200.190:34159 requested a full tablet report, sending...
08:03:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.919902 843 ts_manager.cc:194] Registered new tserver with Master: 2c0c510dd21c40039e7d4798a2c8a019 (127.0.200.131:46651)
08:03:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.920715 1534 heartbeater.cc:344] Connected to a master server at 127.0.200.188:39611
08:03:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.920821 1534 heartbeater.cc:461] Registering TS with master...
08:03:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.921634 1534 heartbeater.cc:507] Master 127.0.200.188:39611 requested a full tablet report, sending...
08:03:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.921663 1536 heartbeater.cc:344] Connected to a master server at 127.0.200.189:44271
08:03:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.921794 1536 heartbeater.cc:461] Registering TS with master...
08:03:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.922197 1536 heartbeater.cc:507] Master 127.0.200.189:44271 requested a full tablet report, sending...
08:03:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.922447 1005 ts_manager.cc:194] Registered new tserver with Master: 2c0c510dd21c40039e7d4798a2c8a019 (127.0.200.131:46651)
08:03:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.923064 925 ts_manager.cc:194] Registered new tserver with Master: 2c0c510dd21c40039e7d4798a2c8a019 (127.0.200.131:46651)
08:03:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.923344 1005 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.0.200.131:45389
08:03:51.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.936267 802 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:03:51.938 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:51.938 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:03:51.939 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.200.190"
port: 34159
}
bound_http_address {
host: "127.0.200.190"
port: 45049
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.200.189"
port: 44271
}
bound_http_address {
host: "127.0.200.189"
port: 32773
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.200.188"
port: 39611
}
bound_http_address {
host: "127.0.200.188"
port: 35305
}
}
}
08:03:51.939 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:03:51.939 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.200.129"
port: 39163
}
bound_http_address {
host: "127.0.200.129"
port: 36863
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.200.130"
port: 41469
}
bound_http_address {
host: "127.0.200.130"
port: 39119
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.200.131"
port: 46651
}
bound_http_address {
host: "127.0.200.131"
port: 43693
}
}
}
08:03:51.940 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:03:51.941 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:03:51.943 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.200.190:34159(127.0.200.190:34159)] connecting to peer
08:03:51.944 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:51.944 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.200.189:44271(127.0.200.189:44271)] connecting to peer
08:03:51.945 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.0.200.190:34159(127.0.200.190:34159)] Successfully connected to peer
08:03:51.945 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.0.200.188:39611(127.0.200.188:39611)] Successfully connected to peer
08:03:51.952 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.0.200.189:44271(127.0.200.189:44271)] Successfully connected to peer
08:03:51.953 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.0.200.188
08:03:51.953 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.0.200.190
08:03:51.956 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.0.200.189
08:03:51.964 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x464ad4d4, L:/127.0.0.1:45764 - R:/127.0.200.188:39611] using SASL/PLAIN
08:03:51.971 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:51.971 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=28), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [23ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, deferred=null) after lookup
08:03:51.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:51.972596 1005 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45764:
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:51.974313 1005 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:51.981 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x1641fce6, L:/127.0.0.1:33202 - R:/127.0.200.190:34159] using SASL/PLAIN
08:03:51.982 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xed47f4ea, L:/127.0.0.1:57724 - R:/127.0.200.189:44271] using SASL/PLAIN
08:03:52.003 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 25a8a9f9357844238befa20212db8067
08:03:52.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.003974 1252 heartbeater.cc:499] Master 127.0.200.188:39611 was elected leader, sending a full tablet report...
08:03:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.050489 1327 tablet_service.cc:1505] Processing CreateTablet for tablet 2be90dfcb5d04b93be51236b1d024232 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.050447 1328 tablet_service.cc:1505] Processing CreateTablet for tablet 687bc4b309484420ad10e1badd1ecd9e (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:52.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.051079 1468 tablet_service.cc:1505] Processing CreateTablet for tablet 2be90dfcb5d04b93be51236b1d024232 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:52.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.050966 1469 tablet_service.cc:1505] Processing CreateTablet for tablet 687bc4b309484420ad10e1badd1ecd9e (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:52.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.052062 1328 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 687bc4b309484420ad10e1badd1ecd9e. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:52.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.052320 1468 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2be90dfcb5d04b93be51236b1d024232. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:52.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.052495 1327 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2be90dfcb5d04b93be51236b1d024232. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:52.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.053454 1469 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 687bc4b309484420ad10e1badd1ecd9e. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:52.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.059001 1187 tablet_service.cc:1505] Processing CreateTablet for tablet 687bc4b309484420ad10e1badd1ecd9e (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:52.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.060366 1186 tablet_service.cc:1505] Processing CreateTablet for tablet 2be90dfcb5d04b93be51236b1d024232 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:52.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.060511 1187 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 687bc4b309484420ad10e1badd1ecd9e. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:52.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.061017 1186 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2be90dfcb5d04b93be51236b1d024232. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.072320 1561 tablet_bootstrap.cc:492] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4: Bootstrap starting.
08:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.072400 1560 tablet_bootstrap.cc:492] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019: Bootstrap starting.
08:03:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.073426 1562 tablet_bootstrap.cc:492] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c: Bootstrap starting.
08:03:52.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.075455 1561 tablet_bootstrap.cc:654] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4: Neither blocks nor log segments found. Creating new log.
08:03:52.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.075631 1560 tablet_bootstrap.cc:654] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019: Neither blocks nor log segments found. Creating new log.
08:03:52.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.076551 1560 log.cc:826] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019: Log is configured to *not* fsync() on all Append() calls
08:03:52.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.076584 1561 log.cc:826] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4: Log is configured to *not* fsync() on all Append() calls
08:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.076958 1562 tablet_bootstrap.cc:654] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c: Neither blocks nor log segments found. Creating new log.
08:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.078085 1562 log.cc:826] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c: Log is configured to *not* fsync() on all Append() calls
08:03:52.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.079001 1561 tablet_bootstrap.cc:492] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4: No bootstrap required, opened a new log
08:03:52.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.079288 1561 ts_tablet_manager.cc:1403] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
08:03:52.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.079348 1560 tablet_bootstrap.cc:492] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019: No bootstrap required, opened a new log
08:03:52.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.079581 1560 ts_tablet_manager.cc:1403] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.004s
08:03:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.081413 1562 tablet_bootstrap.cc:492] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c: No bootstrap required, opened a new log
08:03:52.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.081640 1562 ts_tablet_manager.cc:1403] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c: Time spent bootstrapping tablet: real 0.008s user 0.000s sys 0.006s
08:03:52.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.084581 1561 raft_consensus.cc:359] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.084909 1561 raft_consensus.cc:385] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:52.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.085033 1561 raft_consensus.cc:740] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c8b20a175c74159881815bc68197aa4, State: Initialized, Role: FOLLOWER
08:03:52.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.085629 1561 consensus_queue.cc:260] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.086176 1560 raft_consensus.cc:359] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } }
08:03:52.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.086484 1560 raft_consensus.cc:385] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.086573 1560 raft_consensus.cc:740] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2c0c510dd21c40039e7d4798a2c8a019, State: Initialized, Role: FOLLOWER
08:03:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.086727 1393 heartbeater.cc:499] Master 127.0.200.188:39611 was elected leader, sending a full tablet report...
08:03:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.087229 1561 ts_tablet_manager.cc:1434] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4: Time spent starting tablet: real 0.008s user 0.004s sys 0.005s
08:03:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.087714 1561 tablet_bootstrap.cc:492] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4: Bootstrap starting.
08:03:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.087992 1562 raft_consensus.cc:359] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.087165 1560 consensus_queue.cc:260] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } }
08:03:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.089522 1561 tablet_bootstrap.cc:654] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4: Neither blocks nor log segments found. Creating new log.
08:03:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.090766 1562 raft_consensus.cc:385] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.091012 1562 raft_consensus.cc:740] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83bed13f5bbd49e897e7ee2c1682840c, State: Initialized, Role: FOLLOWER
08:03:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.091836 1562 consensus_queue.cc:260] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.096927 1562 ts_tablet_manager.cc:1434] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c: Time spent starting tablet: real 0.015s user 0.013s sys 0.000s
08:03:52.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.102248 1562 tablet_bootstrap.cc:492] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c: Bootstrap starting.
08:03:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.104461 1562 tablet_bootstrap.cc:654] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c: Neither blocks nor log segments found. Creating new log.
08:03:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.109584 1561 tablet_bootstrap.cc:492] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4: No bootstrap required, opened a new log
08:03:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.109920 1561 ts_tablet_manager.cc:1403] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4: Time spent bootstrapping tablet: real 0.022s user 0.010s sys 0.012s
08:03:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.110780 1534 heartbeater.cc:499] Master 127.0.200.188:39611 was elected leader, sending a full tablet report...
08:03:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.110945 1561 raft_consensus.cc:359] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } }
08:03:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.111116 1561 raft_consensus.cc:385] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.111220 1561 raft_consensus.cc:740] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c8b20a175c74159881815bc68197aa4, State: Initialized, Role: FOLLOWER
08:03:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.111423 1561 consensus_queue.cc:260] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } }
08:03:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.111583 1560 ts_tablet_manager.cc:1434] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019: Time spent starting tablet: real 0.032s user 0.015s sys 0.016s
08:03:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.112098 1560 tablet_bootstrap.cc:492] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019: Bootstrap starting.
08:03:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.113811 1561 ts_tablet_manager.cc:1434] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
08:03:52.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.114166 1560 tablet_bootstrap.cc:654] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019: Neither blocks nor log segments found. Creating new log.
08:03:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.122223 1560 tablet_bootstrap.cc:492] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019: No bootstrap required, opened a new log
08:03:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.122469 1560 ts_tablet_manager.cc:1403] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019: Time spent bootstrapping tablet: real 0.010s user 0.008s sys 0.002s
08:03:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.123409 1560 raft_consensus.cc:359] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.124105 1560 raft_consensus.cc:385] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.124307 1560 raft_consensus.cc:740] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2c0c510dd21c40039e7d4798a2c8a019, State: Initialized, Role: FOLLOWER
08:03:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.124387 1562 tablet_bootstrap.cc:492] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c: No bootstrap required, opened a new log
08:03:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.124508 1562 ts_tablet_manager.cc:1403] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c: Time spent bootstrapping tablet: real 0.022s user 0.000s sys 0.016s
08:03:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.124675 1560 consensus_queue.cc:260] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.125200 1560 ts_tablet_manager.cc:1434] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
08:03:52.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.125607 1562 raft_consensus.cc:359] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } }
08:03:52.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.125769 1562 raft_consensus.cc:385] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:52.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.125847 1562 raft_consensus.cc:740] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83bed13f5bbd49e897e7ee2c1682840c, State: Initialized, Role: FOLLOWER
08:03:52.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.126108 1562 consensus_queue.cc:260] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } }
08:03:52.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.126529 1562 ts_tablet_manager.cc:1434] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
08:03:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:52.169374 1537 tablet.cc:2378] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:52.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:52.230813 1397 tablet.cc:2378] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:52.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:52.252502 1255 tablet.cc:2378] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:52.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.260686 1566 raft_consensus.cc:493] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:52.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.260916 1566 raft_consensus.cc:515] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.262470 1566 leader_election.cc:290] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 83bed13f5bbd49e897e7ee2c1682840c (127.0.200.129:39163), 2c0c510dd21c40039e7d4798a2c8a019 (127.0.200.131:46651)
08:03:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.267187 1569 raft_consensus.cc:493] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.267400 1569 raft_consensus.cc:515] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } }
08:03:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.267884 1567 raft_consensus.cc:493] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.268121 1567 raft_consensus.cc:515] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } }
08:03:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.269098 1489 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2be90dfcb5d04b93be51236b1d024232" candidate_uuid: "9c8b20a175c74159881815bc68197aa4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2c0c510dd21c40039e7d4798a2c8a019" is_pre_election: true
08:03:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.269318 1569 leader_election.cc:290] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2c0c510dd21c40039e7d4798a2c8a019 (127.0.200.131:46651), 9c8b20a175c74159881815bc68197aa4 (127.0.200.130:41469)
08:03:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.269450 1489 raft_consensus.cc:2468] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c8b20a175c74159881815bc68197aa4 in term 0.
08:03:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.269886 1567 leader_election.cc:290] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9c8b20a175c74159881815bc68197aa4 (127.0.200.130:41469), 83bed13f5bbd49e897e7ee2c1682840c (127.0.200.129:39163)
08:03:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.270088 1273 leader_election.cc:304] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2c0c510dd21c40039e7d4798a2c8a019, 9c8b20a175c74159881815bc68197aa4; no voters:
08:03:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.270406 1566 raft_consensus.cc:2804] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.270550 1566 raft_consensus.cc:493] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.270650 1566 raft_consensus.cc:3060] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Advancing to term 1
08:03:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.273620 1566 raft_consensus.cc:515] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.274646 1207 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2be90dfcb5d04b93be51236b1d024232" candidate_uuid: "9c8b20a175c74159881815bc68197aa4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83bed13f5bbd49e897e7ee2c1682840c" is_pre_election: true
08:03:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.275008 1207 raft_consensus.cc:2468] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c8b20a175c74159881815bc68197aa4 in term 0.
08:03:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.275949 1207 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2be90dfcb5d04b93be51236b1d024232" candidate_uuid: "9c8b20a175c74159881815bc68197aa4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83bed13f5bbd49e897e7ee2c1682840c"
08:03:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.276094 1207 raft_consensus.cc:3060] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Advancing to term 1
08:03:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.279209 1207 raft_consensus.cc:2468] T 2be90dfcb5d04b93be51236b1d024232 P 83bed13f5bbd49e897e7ee2c1682840c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c8b20a175c74159881815bc68197aa4 in term 1.
08:03:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.279636 1272 leader_election.cc:304] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 83bed13f5bbd49e897e7ee2c1682840c, 9c8b20a175c74159881815bc68197aa4; no voters:
08:03:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.279733 1206 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "687bc4b309484420ad10e1badd1ecd9e" candidate_uuid: "2c0c510dd21c40039e7d4798a2c8a019" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83bed13f5bbd49e897e7ee2c1682840c" is_pre_election: true
08:03:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.280019 1206 raft_consensus.cc:2468] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2c0c510dd21c40039e7d4798a2c8a019 in term 0.
08:03:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.280411 1566 leader_election.cc:290] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [CANDIDATE]: Term 1 election: Requested vote from peers 83bed13f5bbd49e897e7ee2c1682840c (127.0.200.129:39163), 2c0c510dd21c40039e7d4798a2c8a019 (127.0.200.131:46651)
08:03:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.280472 1413 leader_election.cc:304] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2c0c510dd21c40039e7d4798a2c8a019, 83bed13f5bbd49e897e7ee2c1682840c; no voters:
08:03:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.280596 1566 raft_consensus.cc:2804] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [term 1 FOLLOWER]: Leader election won for term 1
08:03:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.280725 1566 raft_consensus.cc:697] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [term 1 LEADER]: Becoming Leader. State: Replica: 9c8b20a175c74159881815bc68197aa4, State: Running, Role: LEADER
08:03:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.281159 1566 consensus_queue.cc:237] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.282580 1567 raft_consensus.cc:2804] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.282727 1567 raft_consensus.cc:493] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.282810 1567 raft_consensus.cc:3060] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Advancing to term 1
08:03:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.284078 1489 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "687bc4b309484420ad10e1badd1ecd9e" candidate_uuid: "83bed13f5bbd49e897e7ee2c1682840c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2c0c510dd21c40039e7d4798a2c8a019" is_pre_election: true
08:03:52.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.285218 1348 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "687bc4b309484420ad10e1badd1ecd9e" candidate_uuid: "83bed13f5bbd49e897e7ee2c1682840c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c8b20a175c74159881815bc68197aa4" is_pre_election: true
08:03:52.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.285537 1348 raft_consensus.cc:2468] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 83bed13f5bbd49e897e7ee2c1682840c in term 0.
08:03:52.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.286015 1132 leader_election.cc:304] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 83bed13f5bbd49e897e7ee2c1682840c, 9c8b20a175c74159881815bc68197aa4; no voters:
08:03:52.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.286406 1569 raft_consensus.cc:2804] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:52.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.286537 1569 raft_consensus.cc:493] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:52.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.286641 1569 raft_consensus.cc:3060] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Advancing to term 1
08:03:52.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.290117 1569 raft_consensus.cc:515] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } }
08:03:52.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.291050 1488 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2be90dfcb5d04b93be51236b1d024232" candidate_uuid: "9c8b20a175c74159881815bc68197aa4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2c0c510dd21c40039e7d4798a2c8a019"
08:03:52.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.291304 1488 raft_consensus.cc:3060] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Advancing to term 1
08:03:52.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.291330 1487 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "687bc4b309484420ad10e1badd1ecd9e" candidate_uuid: "83bed13f5bbd49e897e7ee2c1682840c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2c0c510dd21c40039e7d4798a2c8a019"
08:03:52.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.291473 1348 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "687bc4b309484420ad10e1badd1ecd9e" candidate_uuid: "2c0c510dd21c40039e7d4798a2c8a019" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c8b20a175c74159881815bc68197aa4" is_pre_election: true
08:03:52.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.291646 1348 raft_consensus.cc:2468] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2c0c510dd21c40039e7d4798a2c8a019 in term 0.
08:03:52.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.292559 1569 leader_election.cc:290] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [CANDIDATE]: Term 1 election: Requested vote from peers 2c0c510dd21c40039e7d4798a2c8a019 (127.0.200.131:46651), 9c8b20a175c74159881815bc68197aa4 (127.0.200.130:41469)
08:03:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.292742 1348 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "687bc4b309484420ad10e1badd1ecd9e" candidate_uuid: "83bed13f5bbd49e897e7ee2c1682840c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c8b20a175c74159881815bc68197aa4"
08:03:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.292973 1348 raft_consensus.cc:3060] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Advancing to term 1
08:03:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.294134 1488 raft_consensus.cc:2468] T 2be90dfcb5d04b93be51236b1d024232 P 2c0c510dd21c40039e7d4798a2c8a019 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c8b20a175c74159881815bc68197aa4 in term 1.
08:03:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.294096 1567 raft_consensus.cc:515] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } }
08:03:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.294456 1489 raft_consensus.cc:2393] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 83bed13f5bbd49e897e7ee2c1682840c in current term 1: Already voted for candidate 2c0c510dd21c40039e7d4798a2c8a019 in this term.
08:03:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.297003 1347 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "687bc4b309484420ad10e1badd1ecd9e" candidate_uuid: "2c0c510dd21c40039e7d4798a2c8a019" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c8b20a175c74159881815bc68197aa4"
08:03:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.297228 1207 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "687bc4b309484420ad10e1badd1ecd9e" candidate_uuid: "2c0c510dd21c40039e7d4798a2c8a019" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83bed13f5bbd49e897e7ee2c1682840c"
08:03:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.297468 1207 raft_consensus.cc:2393] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 2c0c510dd21c40039e7d4798a2c8a019 in current term 1: Already voted for candidate 83bed13f5bbd49e897e7ee2c1682840c in this term.
08:03:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.297811 1567 leader_election.cc:290] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [CANDIDATE]: Term 1 election: Requested vote from peers 9c8b20a175c74159881815bc68197aa4 (127.0.200.130:41469), 83bed13f5bbd49e897e7ee2c1682840c (127.0.200.129:39163)
08:03:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.298300 1413 leader_election.cc:304] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2c0c510dd21c40039e7d4798a2c8a019; no voters: 83bed13f5bbd49e897e7ee2c1682840c, 9c8b20a175c74159881815bc68197aa4
08:03:52.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.298686 1567 raft_consensus.cc:2749] T 687bc4b309484420ad10e1badd1ecd9e P 2c0c510dd21c40039e7d4798a2c8a019 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
08:03:52.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.298820 1348 raft_consensus.cc:2468] T 687bc4b309484420ad10e1badd1ecd9e P 9c8b20a175c74159881815bc68197aa4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 83bed13f5bbd49e897e7ee2c1682840c in term 1.
08:03:52.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.299244 1132 leader_election.cc:304] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 83bed13f5bbd49e897e7ee2c1682840c, 9c8b20a175c74159881815bc68197aa4; no voters: 2c0c510dd21c40039e7d4798a2c8a019
08:03:52.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.299649 1569 raft_consensus.cc:2804] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 1 FOLLOWER]: Leader election won for term 1
08:03:52.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.300120 1569 raft_consensus.cc:697] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [term 1 LEADER]: Becoming Leader. State: Replica: 83bed13f5bbd49e897e7ee2c1682840c, State: Running, Role: LEADER
08:03:52.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.300158 1006 catalog_manager.cc:5649] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9c8b20a175c74159881815bc68197aa4 (127.0.200.130). New cstate: current_term: 1 leader_uuid: "9c8b20a175c74159881815bc68197aa4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } health_report { overall_health: UNKNOWN } } }
08:03:52.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.300633 1569 consensus_queue.cc:237] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } }
08:03:52.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.305984 1005 catalog_manager.cc:5649] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c reported cstate change: term changed from 0 to 1, leader changed from <none> to 83bed13f5bbd49e897e7ee2c1682840c (127.0.200.129). New cstate: current_term: 1 leader_uuid: "83bed13f5bbd49e897e7ee2c1682840c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } health_report { overall_health: HEALTHY } } }
08:03:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.403550 1005 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45770:
08:03:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:03:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:52.404407 1005 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.415422 1187 tablet_service.cc:1505] Processing CreateTablet for tablet 39069dd33ea449b28c95ff20e283e6d7 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=732382f1fe234432b85505b59925bd05]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.415366 1469 tablet_service.cc:1505] Processing CreateTablet for tablet 39069dd33ea449b28c95ff20e283e6d7 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=732382f1fe234432b85505b59925bd05]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.416273 1187 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 39069dd33ea449b28c95ff20e283e6d7. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.416277 1469 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 39069dd33ea449b28c95ff20e283e6d7. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.415948 1327 tablet_service.cc:1505] Processing CreateTablet for tablet 39069dd33ea449b28c95ff20e283e6d7 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=732382f1fe234432b85505b59925bd05]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.416641 1327 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 39069dd33ea449b28c95ff20e283e6d7. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:52.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.421200 1562 tablet_bootstrap.cc:492] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c: Bootstrap starting.
08:03:52.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.422662 1562 tablet_bootstrap.cc:654] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c: Neither blocks nor log segments found. Creating new log.
08:03:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.423377 1561 tablet_bootstrap.cc:492] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: Bootstrap starting.
08:03:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.424351 1562 tablet_bootstrap.cc:492] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c: No bootstrap required, opened a new log
08:03:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.424539 1562 ts_tablet_manager.cc:1403] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
08:03:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.424819 1561 tablet_bootstrap.cc:654] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: Neither blocks nor log segments found. Creating new log.
08:03:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.425161 1562 raft_consensus.cc:359] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.425283 1562 raft_consensus.cc:385] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.425340 1562 raft_consensus.cc:740] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83bed13f5bbd49e897e7ee2c1682840c, State: Initialized, Role: FOLLOWER
08:03:52.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.425554 1562 consensus_queue.cc:260] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.425896 1569 tablet_replica.cc:442] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } }
08:03:52.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.426074 1569 tablet_replica.cc:445] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:52.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.426797 1561 tablet_bootstrap.cc:492] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: No bootstrap required, opened a new log
08:03:52.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.427047 1561 ts_tablet_manager.cc:1403] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
08:03:52.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.427886 1561 raft_consensus.cc:359] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.428098 1561 raft_consensus.cc:385] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:52.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.428179 1561 raft_consensus.cc:740] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c8b20a175c74159881815bc68197aa4, State: Initialized, Role: FOLLOWER
08:03:52.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.428449 1561 consensus_queue.cc:260] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.428578 1562 ts_tablet_manager.cc:1434] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c: Time spent starting tablet: real 0.004s user 0.000s sys 0.001s
08:03:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.428900 1566 tablet_replica.cc:442] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } }
08:03:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.429138 1566 tablet_replica.cc:445] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.430475 1561 ts_tablet_manager.cc:1434] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
08:03:52.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.434619 1560 tablet_bootstrap.cc:492] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019: Bootstrap starting.
08:03:52.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.436096 1560 tablet_bootstrap.cc:654] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019: Neither blocks nor log segments found. Creating new log.
08:03:52.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.437407 1560 tablet_bootstrap.cc:492] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019: No bootstrap required, opened a new log
08:03:52.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.437544 1560 ts_tablet_manager.cc:1403] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
08:03:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.438197 1560 raft_consensus.cc:359] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.438395 1560 raft_consensus.cc:385] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.438452 1560 raft_consensus.cc:740] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2c0c510dd21c40039e7d4798a2c8a019, State: Initialized, Role: FOLLOWER
08:03:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.438685 1560 consensus_queue.cc:260] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.439170 1560 ts_tablet_manager.cc:1434] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
08:03:52.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.439111 1567 tablet_replica.cc:442] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } }
08:03:52.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.439318 1567 tablet_replica.cc:445] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.621814 1566 raft_consensus.cc:493] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.621974 1566 raft_consensus.cc:515] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.622426 1566 leader_election.cc:290] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 83bed13f5bbd49e897e7ee2c1682840c (127.0.200.129:39163), 2c0c510dd21c40039e7d4798a2c8a019 (127.0.200.131:46651)
08:03:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.622839 1207 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "39069dd33ea449b28c95ff20e283e6d7" candidate_uuid: "9c8b20a175c74159881815bc68197aa4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83bed13f5bbd49e897e7ee2c1682840c" is_pre_election: true
08:03:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.622900 1489 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "39069dd33ea449b28c95ff20e283e6d7" candidate_uuid: "9c8b20a175c74159881815bc68197aa4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2c0c510dd21c40039e7d4798a2c8a019" is_pre_election: true
08:03:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.623030 1207 raft_consensus.cc:2468] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c8b20a175c74159881815bc68197aa4 in term 0.
08:03:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.623087 1489 raft_consensus.cc:2468] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c8b20a175c74159881815bc68197aa4 in term 0.
08:03:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.623380 1272 leader_election.cc:304] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 83bed13f5bbd49e897e7ee2c1682840c, 9c8b20a175c74159881815bc68197aa4; no voters:
08:03:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.623662 1566 raft_consensus.cc:2804] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.623737 1566 raft_consensus.cc:493] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:52.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.623797 1566 raft_consensus.cc:3060] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [term 0 FOLLOWER]: Advancing to term 1
08:03:52.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.625422 1566 raft_consensus.cc:515] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.625941 1566 leader_election.cc:290] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [CANDIDATE]: Term 1 election: Requested vote from peers 83bed13f5bbd49e897e7ee2c1682840c (127.0.200.129:39163), 2c0c510dd21c40039e7d4798a2c8a019 (127.0.200.131:46651)
08:03:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.626250 1207 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "39069dd33ea449b28c95ff20e283e6d7" candidate_uuid: "9c8b20a175c74159881815bc68197aa4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83bed13f5bbd49e897e7ee2c1682840c"
08:03:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.626291 1489 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "39069dd33ea449b28c95ff20e283e6d7" candidate_uuid: "9c8b20a175c74159881815bc68197aa4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2c0c510dd21c40039e7d4798a2c8a019"
08:03:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.626406 1489 raft_consensus.cc:3060] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019 [term 0 FOLLOWER]: Advancing to term 1
08:03:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.626415 1207 raft_consensus.cc:3060] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c [term 0 FOLLOWER]: Advancing to term 1
08:03:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.628140 1489 raft_consensus.cc:2468] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c8b20a175c74159881815bc68197aa4 in term 1.
08:03:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.628417 1207 raft_consensus.cc:2468] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c8b20a175c74159881815bc68197aa4 in term 1.
08:03:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.628502 1273 leader_election.cc:304] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2c0c510dd21c40039e7d4798a2c8a019, 9c8b20a175c74159881815bc68197aa4; no voters:
08:03:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.628724 1566 raft_consensus.cc:2804] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [term 1 FOLLOWER]: Leader election won for term 1
08:03:52.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.628883 1566 raft_consensus.cc:697] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [term 1 LEADER]: Becoming Leader. State: Replica: 9c8b20a175c74159881815bc68197aa4, State: Running, Role: LEADER
08:03:52.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.629191 1566 consensus_queue.cc:237] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } }
08:03:52.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.630519 1577 tablet_replica.cc:442] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: TxnStatusTablet state changed. Reason: New leader 9c8b20a175c74159881815bc68197aa4. Latest consensus state: current_term: 1 leader_uuid: "9c8b20a175c74159881815bc68197aa4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } }
08:03:52.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.630813 1577 tablet_replica.cc:445] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: This TxnStatusTablet replica's current role is: LEADER
08:03:52.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.631408 1593 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
08:03:52.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.631844 1593 txn_status_manager.cc:930] Loading transaction status metadata into memory...
08:03:52.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.632467 1004 catalog_manager.cc:5649] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9c8b20a175c74159881815bc68197aa4 (127.0.200.130). New cstate: current_term: 1 leader_uuid: "9c8b20a175c74159881815bc68197aa4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } health_report { overall_health: UNKNOWN } } }
08:03:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.669270 1207 raft_consensus.cc:1275] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c [term 1 FOLLOWER]: Refusing update from remote peer 9c8b20a175c74159881815bc68197aa4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.669252 1489 raft_consensus.cc:1275] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019 [term 1 FOLLOWER]: Refusing update from remote peer 9c8b20a175c74159881815bc68197aa4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:52.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.670061 1577 consensus_queue.cc:1048] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:52.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.670336 1566 consensus_queue.cc:1048] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:52.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.682808 1567 tablet_replica.cc:442] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019: TxnStatusTablet state changed. Reason: New leader 9c8b20a175c74159881815bc68197aa4. Latest consensus state: current_term: 1 leader_uuid: "9c8b20a175c74159881815bc68197aa4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } }
08:03:52.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.683034 1567 tablet_replica.cc:445] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:52.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.686607 1596 mvcc.cc:204] Tried to move back new op lower bound from 7214610156204826624 to 7214610156050874368. Current Snapshot: MvccSnapshot[applied={T|T < 7214610156204826624}]
08:03:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.688475 1569 tablet_replica.cc:442] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c: TxnStatusTablet state changed. Reason: New leader 9c8b20a175c74159881815bc68197aa4. Latest consensus state: current_term: 1 leader_uuid: "9c8b20a175c74159881815bc68197aa4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } }
08:03:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.688740 1569 tablet_replica.cc:445] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.690696 1566 tablet_replica.cc:442] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9c8b20a175c74159881815bc68197aa4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } }
08:03:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.690955 1566 tablet_replica.cc:445] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: This TxnStatusTablet replica's current role is: LEADER
08:03:52.694 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
08:03:52.698 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 25a8a9f9357844238befa20212db8067
08:03:52.698 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.200.188:39611
08:03:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.697738 1566 tablet_replica.cc:442] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9c8b20a175c74159881815bc68197aa4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } }
08:03:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.697964 1566 tablet_replica.cc:445] T 39069dd33ea449b28c95ff20e283e6d7 P 9c8b20a175c74159881815bc68197aa4: This TxnStatusTablet replica's current role is: LEADER
08:03:52.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:03:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.702422 802 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu pid 973
08:03:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.704645 1046 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.706044 1569 tablet_replica.cc:442] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9c8b20a175c74159881815bc68197aa4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } }
08:03:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.706182 1569 tablet_replica.cc:445] T 39069dd33ea449b28c95ff20e283e6d7 P 83bed13f5bbd49e897e7ee2c1682840c: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:52.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.709806 1567 tablet_replica.cc:442] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9c8b20a175c74159881815bc68197aa4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 } } peers { permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 } } peers { permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 } } }
08:03:52.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.710011 1567 tablet_replica.cc:445] T 39069dd33ea449b28c95ff20e283e6d7 P 2c0c510dd21c40039e7d4798a2c8a019: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.823027 1577 consensus_queue.cc:1048] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" member_type: VOTER last_known_addr { host: "127.0.200.129" port: 39163 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.828577 1577 consensus_queue.cc:1048] T 2be90dfcb5d04b93be51236b1d024232 P 9c8b20a175c74159881815bc68197aa4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:52.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.834868 802 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu with pid 973
08:03:52.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.861755 1569 consensus_queue.cc:1048] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c8b20a175c74159881815bc68197aa4" member_type: VOTER last_known_addr { host: "127.0.200.130" port: 41469 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:52.865 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.0.200.188:39611(127.0.200.188:39611)] handling channelInactive
08:03:52.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state READY due to: connection closed
08:03:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:52.866221 1415 connection.cc:537] client connection to 127.0.200.188:39611 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:52.866648 1252 heartbeater.cc:646] Failed to heartbeat to 127.0.200.188:39611 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.0.200.188:39611 (error 108)
08:03:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:52.867329 1393 heartbeater.cc:646] Failed to heartbeat to 127.0.200.188:39611 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.0.200.188:39611 (error 108)
08:03:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:52.868121 1534 heartbeater.cc:646] Failed to heartbeat to 127.0.200.188:39611 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:52.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:03:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:52.880375 1611 negotiation.cc:337] Failed RPC negotiation. Trace:
08:03:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:52.868643 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.200.188:39611 (local address 127.0.200.131:43505)
08:03:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:52.879116 (+ 10473us) negotiation.cc:107] Waiting for socket to connect
08:03:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:52.879223 (+ 107us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection reset by peer (error 104)
08:03:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":10329,"thread_start_us":10221,"threads_started":1}
08:03:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:52.880961 1609 negotiation.cc:337] Failed RPC negotiation. Trace:
08:03:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:52.867152 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.200.188:39611 (local address 127.0.200.129:47695)
08:03:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:52.879024 (+ 11872us) negotiation.cc:107] Waiting for socket to connect
08:03:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:52.879196 (+ 172us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection reset by peer (error 104)
08:03:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":11738,"threads_started":1}
08:03:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:52.882344 1610 negotiation.cc:337] Failed RPC negotiation. Trace:
08:03:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:52.868643 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.200.188:39611 (local address 127.0.200.130:47941)
08:03:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:52.880988 (+ 12345us) negotiation.cc:107] Waiting for socket to connect
08:03:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:52.881667 (+ 679us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection reset by peer (error 104)
08:03:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":12234,"thread_start_us":12450,"threads_started":1}
08:03:52.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:52.887636 1608 consensus_queue.cc:1048] T 687bc4b309484420ad10e1badd1ecd9e P 83bed13f5bbd49e897e7ee2c1682840c [LEADER]: Connected to new peer: Peer: permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" member_type: VOTER last_known_addr { host: "127.0.200.131" port: 46651 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:53.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.133348 1616 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 4c116f66397249a491b1d9259e219594)
08:03:53.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.133558 1616 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.134433 1617 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 4c116f66397249a491b1d9259e219594)
08:03:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.134620 1617 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:53.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.135272 1616 leader_election.cc:290] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9809071a62ca479b8178425bef946094 (127.0.200.189:44271), 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611)
08:03:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.135565 934 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9ed5e67dc759449b960c2c71ee039c42" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "9809071a62ca479b8178425bef946094" is_pre_election: true
08:03:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.135831 934 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9ed5e67dc759449b960c2c71ee039c42 in term 1.
08:03:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.136708 853 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9809071a62ca479b8178425bef946094" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "9ed5e67dc759449b960c2c71ee039c42" is_pre_election: true
08:03:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.136904 1617 leader_election.cc:290] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159), 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611)
08:03:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.136971 853 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9809071a62ca479b8178425bef946094 in term 1.
08:03:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.137477 830 leader_election.cc:304] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9809071a62ca479b8178425bef946094, 9ed5e67dc759449b960c2c71ee039c42; no voters:
08:03:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.137890 1616 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:03:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.138032 1616 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 4c116f66397249a491b1d9259e219594)
08:03:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.138135 1616 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 1 FOLLOWER]: Advancing to term 2
08:03:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.138101 912 leader_election.cc:304] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9809071a62ca479b8178425bef946094, 9ed5e67dc759449b960c2c71ee039c42; no voters:
08:03:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:53.138465 911 leader_election.cc:336] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611): Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:53.138526 830 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611): Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.138702 1617 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:03:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.138813 1617 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 4c116f66397249a491b1d9259e219594)
08:03:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.138906 1617 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 1 FOLLOWER]: Advancing to term 2
08:03:53.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.140502 1616 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:53.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.140923 1616 leader_election.cc:290] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 2 election: Requested vote from peers 9809071a62ca479b8178425bef946094 (127.0.200.189:44271), 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611)
08:03:53.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.141116 1617 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:53.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.141280 934 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9ed5e67dc759449b960c2c71ee039c42" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "9809071a62ca479b8178425bef946094"
08:03:53.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.141520 934 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 9ed5e67dc759449b960c2c71ee039c42 in current term 2: Already voted for candidate 9809071a62ca479b8178425bef946094 in this term.
08:03:53.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.141764 1617 leader_election.cc:290] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [CANDIDATE]: Term 2 election: Requested vote from peers 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159), 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611)
08:03:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.142450 853 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9809071a62ca479b8178425bef946094" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "9ed5e67dc759449b960c2c71ee039c42"
08:03:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.142706 853 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 9809071a62ca479b8178425bef946094 in current term 2: Already voted for candidate 9ed5e67dc759449b960c2c71ee039c42 in this term.
08:03:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:53.142751 830 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611): Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:53.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.142918 830 leader_election.cc:304] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9ed5e67dc759449b960c2c71ee039c42; no voters: 4c116f66397249a491b1d9259e219594, 9809071a62ca479b8178425bef946094
08:03:53.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:53.143028 911 leader_election.cc:336] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611): Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:53.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.143201 1616 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
08:03:53.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.143445 912 leader_election.cc:304] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9809071a62ca479b8178425bef946094; no voters: 4c116f66397249a491b1d9259e219594, 9ed5e67dc759449b960c2c71ee039c42
08:03:53.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.143728 1617 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
08:03:53.204 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:53.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.205 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.0.200.188:39611(127.0.200.188:39611) for tablet Kudu Master: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.223 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:53.224 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.224 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.224 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.188:39611: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.225 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:53.243 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:53.243 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.243 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.244 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.188:39611: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.244 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:53.263 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:53.264 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.267 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.188:39611: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.267 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:53.283 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:53.284 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.284 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.284 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.188:39611: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.284 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:53.323 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:53.324 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.324 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.324 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.188:39611: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.325 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:53.343 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:53.344 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.344 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.344 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.188:39611: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.344 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:53.443 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:53.444 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.444 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.444 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.188:39611: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.445 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:53.446 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] sending RPC to server master-127.0.200.188:39611, [1ms] received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [18ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [20ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [20ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [38ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [40ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [40ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [40ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [62ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [63ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [80ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [80ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [80ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [80ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [120ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [120ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [121ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [140ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [140ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [140ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [240ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [241ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [241ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, deferred=Deferred@483270647(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1771505555@34b0bebe -> (continuation of Deferred@1921579137 after retry RPC after error@534840570) -> (continuation of Deferred@1933349206 after retry RPC after error@1120437441) -> (continuation of Deferred@2008101097 after retry RPC after error@1616915505) -> (continuation of Deferred@2009184074 after retry RPC after error@1898568703) -> (continuation of Deferred@266443595 after retry RPC after error@1853959970) -> (continuation of Deferred@343977567 after retry RPC after error@726231315), errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/263344142@28f184f2 -> (continuation of Deferred@1921579137 after retry RPC after error@534840570) -> (continuation of Deferred@1933349206 after retry RPC after error@1120437441) -> (continuation of Deferred@2008101097 after retry RPC after error@1616915505) -> (continuation of Deferred@2009184074 after retry RPC after error@1898568703) -> (continuation of Deferred@266443595 after retry RPC after error@1853959970) -> (continuation of Deferred@343977567 after retry RPC after error@726231315)))
08:03:53.446 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] sending RPC to server master-127.0.200.188:39611, [1ms] received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [18ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [20ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [20ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [38ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [40ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [40ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [40ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [62ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [63ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [80ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [80ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [80ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [80ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [120ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [120ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [121ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [140ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [140ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [140ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [240ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [241ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [241ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, deferred=Deferred@483270647(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1771505555@34b0bebe -> (continuation of Deferred@1921579137 after retry RPC after error@534840570) -> (continuation of Deferred@1933349206 after retry RPC after error@1120437441) -> (continuation of Deferred@2008101097 after retry RPC after error@1616915505) -> (continuation of Deferred@2009184074 after retry RPC after error@1898568703) -> (continuation of Deferred@266443595 after retry RPC after error@1853959970) -> (continuation of Deferred@343977567 after retry RPC after error@726231315), errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/263344142@28f184f2 -> (continuation of Deferred@1921579137 after retry RPC after error@534840570) -> (continuation of Deferred@1933349206 after retry RPC after error@1120437441) -> (continuation of Deferred@2008101097 after retry RPC after error@1616915505) -> (continuation of Deferred@2009184074 after retry RPC after error@1898568703) -> (continuation of Deferred@266443595 after retry RPC after error@1853959970) -> (continuation of Deferred@343977567 after retry RPC after error@726231315)))
08:03:53.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.451882 1616 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:53.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.452033 1616 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:53.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.452502 1616 leader_election.cc:290] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 9809071a62ca479b8178425bef946094 (127.0.200.189:44271), 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611)
08:03:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.452881 934 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9ed5e67dc759449b960c2c71ee039c42" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "9809071a62ca479b8178425bef946094" is_pre_election: true
08:03:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.453101 934 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9ed5e67dc759449b960c2c71ee039c42 in term 2.
08:03:53.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.453693 830 leader_election.cc:304] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9809071a62ca479b8178425bef946094, 9ed5e67dc759449b960c2c71ee039c42; no voters:
08:03:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.453969 1616 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 2 FOLLOWER]: Leader pre-election won for term 3
08:03:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.454058 1616 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.454164 1616 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 2 FOLLOWER]: Advancing to term 3
08:03:53.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.456579 1616 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:53.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.457063 1616 leader_election.cc:290] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 3 election: Requested vote from peers 9809071a62ca479b8178425bef946094 (127.0.200.189:44271), 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611)
08:03:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:53.457460 830 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611): Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.458276 934 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9ed5e67dc759449b960c2c71ee039c42" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "9809071a62ca479b8178425bef946094"
08:03:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.458442 934 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 2 FOLLOWER]: Advancing to term 3
08:03:53.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:53.459848 830 leader_election.cc:336] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611): Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111)
08:03:53.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.461535 934 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9ed5e67dc759449b960c2c71ee039c42 in term 3.
08:03:53.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.461866 830 leader_election.cc:304] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9809071a62ca479b8178425bef946094, 9ed5e67dc759449b960c2c71ee039c42; no voters: 4c116f66397249a491b1d9259e219594
08:03:53.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.462070 1616 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 3 FOLLOWER]: Leader election won for term 3
08:03:53.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.462385 1616 raft_consensus.cc:697] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [term 3 LEADER]: Becoming Leader. State: Replica: 9ed5e67dc759449b960c2c71ee039c42, State: Running, Role: LEADER
08:03:53.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.462764 1616 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:53.463 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:53.464 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.464 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.464 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.188:39611: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.465034 1628 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9ed5e67dc759449b960c2c71ee039c42. Latest consensus state: current_term: 3 leader_uuid: "9ed5e67dc759449b960c2c71ee039c42" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:53.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.465260 1628 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [sys.catalog]: This master's current role is: LEADER
08:03:53.465 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:53.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.466529 1629 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:03:53.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.469048 1629 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067]
08:03:53.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.469506 1629 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=732382f1fe234432b85505b59925bd05]
08:03:53.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.470904 1629 tablet_loader.cc:96] loaded metadata for tablet 2be90dfcb5d04b93be51236b1d024232 (table txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067])
08:03:53.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.471248 1629 tablet_loader.cc:96] loaded metadata for tablet 39069dd33ea449b28c95ff20e283e6d7 (table kudu_system.kudu_transactions [id=732382f1fe234432b85505b59925bd05])
08:03:53.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.471635 1629 tablet_loader.cc:96] loaded metadata for tablet 687bc4b309484420ad10e1badd1ecd9e (table txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067])
08:03:53.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.471896 1629 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:03:53.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.472625 1629 catalog_manager.cc:1269] Loaded cluster ID: 11103c47897d43f1bc346107e55096ee
08:03:53.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.472720 1629 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:03:53.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.473711 1629 catalog_manager.cc:1514] Loading token signing keys...
08:03:53.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.474428 1629 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42: Loaded TSK: 0
08:03:53.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.475153 1629 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:03:53.483 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:53.484 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.484 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.484 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.188:39611: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611
08:03:53.486 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:53.486 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=2, TimeoutTracker(timeout=250, elapsed=22), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [1ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [1ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [21ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: Network error: Failed to connect to peer master-127.0.200.188:39611(127.0.200.188:39611): Connection refused: /127.0.200.188:39611, [22ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: OK, deferred=Deferred@662356321(state=PENDING, result=null, callback=(continuation of Deferred@371086168 after retry RPC after error@1219337245), errback=(continuation of Deferred@371086168 after retry RPC after error@1219337245))) after lookup
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.526243 844 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33224:
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:53.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:53.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:03:53.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:53.528275 844 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:53.568051 830 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 -> Peer 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611): Couldn't send request to peer 4c116f66397249a491b1d9259e219594. Status: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:53.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.579277 934 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 3 FOLLOWER]: Refusing update from remote peer 9ed5e67dc759449b960c2c71ee039c42: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
08:03:53.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.579900 1628 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
08:03:53.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.581293 1617 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9ed5e67dc759449b960c2c71ee039c42. Latest consensus state: current_term: 3 leader_uuid: "9ed5e67dc759449b960c2c71ee039c42" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:53.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.581543 1617 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: This master's current role is: FOLLOWER
08:03:53.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.582755 1616 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "9ed5e67dc759449b960c2c71ee039c42" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:53.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.582947 1616 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 [sys.catalog]: This master's current role is: LEADER
08:03:53.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.583307 1617 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "9ed5e67dc759449b960c2c71ee039c42" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:53.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:53.583474 1617 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: This master's current role is: FOLLOWER
08:03:54.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:54.063053 830 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 -> Peer 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611): Couldn't send request to peer 4c116f66397249a491b1d9259e219594. Status: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:03:54.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:54.442327 830 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111) [suppressed 20 similar messages]
08:03:54.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:54.554270 830 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 -> Peer 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611): Couldn't send request to peer 4c116f66397249a491b1d9259e219594. Status: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
08:03:54.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 687bc4b309484420ad10e1badd1ecd9e for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
08:03:54.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 2be90dfcb5d04b93be51236b1d024232 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
08:03:54.880 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=687bc4b309484420ad10e1badd1ecd9e@[2c0c510dd21c40039e7d4798a2c8a019(127.0.200.131:46651),83bed13f5bbd49e897e7ee2c1682840c(127.0.200.129:39163)[L],9c8b20a175c74159881815bc68197aa4(127.0.200.130:41469)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=2be90dfcb5d04b93be51236b1d024232@[2c0c510dd21c40039e7d4798a2c8a019(127.0.200.131:46651),83bed13f5bbd49e897e7ee2c1682840c(127.0.200.129:39163),9c8b20a175c74159881815bc68197aa4(127.0.200.130:41469)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
08:03:54.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.0.200.190:34159, [2ms] Sub RPC GetTableLocations: received response from server master-127.0.200.190:34159: OK, deferred=null) after lookup
08:03:54.881 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 83bed13f5bbd49e897e7ee2c1682840c(127.0.200.129:39163)] connecting to peer
08:03:54.881 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 83bed13f5bbd49e897e7ee2c1682840c(127.0.200.129:39163)] Successfully connected to peer
08:03:54.891 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x1df1ee7f, L:/127.0.0.1:56194 - R:/127.0.200.129:39163] using TOKEN/null
08:03:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:54.895231 1162 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.0.200.188:39611)
08:03:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:54.896306 1535 heartbeater.cc:499] Master 127.0.200.190:34159 was elected leader, sending a full tablet report...
08:03:54.939 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.200.188:39611
08:03:54.939 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:54.941843 802 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/logs
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data/info.pb
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.188:39611
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.188
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35305
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43367/jwks.json
08:03:54.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:54.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:55.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.023877 830 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9ed5e67dc759449b960c2c71ee039c42 -> Peer 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611): Couldn't send request to peer 4c116f66397249a491b1d9259e219594. Status: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
08:03:55.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.240449 1668 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:55.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.240701 1668 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:55.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.240777 1668 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.247566 1668 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.247654 1668 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.247725 1668 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.247792 1668 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.247851 1668 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.247896 1668 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.247946 1668 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43367/jwks.json
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.256275 1668 master_runner.cc:386] Master server non-default flags:
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.200.148:44969
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.200.190:34159,127.0.200.189:44271,127.0.200.188:39611
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.200.188:39611
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43367/jwks.json
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data/info.pb
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.200.188
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35305
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/logs
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d0cb013a5aad754f2ed06c25f5ec3d8a1873622
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Oct 2025 07:43:22 UTC on 24a791456cd2
08:03:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8708
08:03:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.257711 1668 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.259364 1668 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.266300 1674 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.266391 1673 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:55.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.267160 1676 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:55.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.267532 1668 server_base.cc:1047] running on GCE node
08:03:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.268137 1668 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.269304 1668 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:55.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.270506 1668 hybrid_clock.cc:648] HybridClock initialized: now 1761379435270446 us; error 43 us; skew 500 ppm
08:03:55.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.270749 1668 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.272651 1668 webserver.cc:492] Webserver started at http://127.0.200.188:35305/ using document root <none> and password file <none>
08:03:55.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.273226 1668 fs_manager.cc:362] Metadata directory not provided
08:03:55.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.273322 1668 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.277611 1668 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.004s sys 0.000s
08:03:55.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.280143 1682 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:55.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.281216 1668 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
08:03:55.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.281340 1668 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data,/tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal
08:03:55.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c116f66397249a491b1d9259e219594"
08:03:55.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-25 08:03:50 on dist-test-slave-37ck"
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.281898 1668 fs_report.cc:389] FS layout report
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/wal
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data/data
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.296483 1668 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:55.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.297093 1668 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:55.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.297283 1668 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:55.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.302827 813 master_runner.cc:186] Error getting master registration for 127.0.200.188:39611: Network error: Client connection negotiation failed: client connection to 127.0.200.188:39611: connect: Connection refused (error 111),
08:03:55.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.303032 813 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.317516 1668 rpc_server.cc:307] RPC server started. Bound to: 127.0.200.188:39611
08:03:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.317533 1743 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.200.188:39611 every 8 connection(s)
08:03:55.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.318915 1668 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWhNLRi/test-tmp/mini-kudu-cluster1290105095160255434/master-2/data/info.pb
08:03:55.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.324931 1744 sys_catalog.cc:263] Verifying existing consensus state
08:03:55.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.326928 802 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu as pid 1668
08:03:55.327 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:03:55.327 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.200.190:34159
08:03:55.327 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:03:55.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.327955 1744 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: Bootstrap starting.
08:03:55.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.327968 802 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu pid 813
08:03:55.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.328920 884 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:55.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.347196 1744 log.cc:826] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: Log is configured to *not* fsync() on all Append() calls
08:03:55.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.355966 1394 heartbeater.cc:499] Master 127.0.200.190:34159 was elected leader, sending a full tablet report...
08:03:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.363736 1744 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.364152 1744 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: Bootstrap complete.
08:03:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.369441 1744 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.370002 1744 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4c116f66397249a491b1d9259e219594, State: Initialized, Role: FOLLOWER
08:03:55.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.371031 1744 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.372812 1750 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.373055 1750 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: This master's current role is: FOLLOWER
08:03:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.373817 1744 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: configured and running, proceeding with master startup.
08:03:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.374521 1393 heartbeater.cc:344] Connected to a master server at 127.0.200.188:39611
08:03:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.386035 1761 catalog_manager.cc:1269] Loaded cluster ID: 11103c47897d43f1bc346107e55096ee
08:03:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.386154 1761 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: loading cluster ID for follower catalog manager: success
08:03:55.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.389665 1761 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: acquiring CA information for follower catalog manager: success
08:03:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.391680 1761 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:55.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.461295 1253 heartbeater.cc:499] Master 127.0.200.190:34159 was elected leader, sending a full tablet report...
08:03:55.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.470510 1709 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 1 FOLLOWER]: Advancing to term 3
08:03:55.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.477283 802 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu with pid 813
08:03:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.486532 1770 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "9ed5e67dc759449b960c2c71ee039c42" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.486919 1770 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: This master's current role is: FOLLOWER
08:03:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.487454 1750 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9ed5e67dc759449b960c2c71ee039c42. Latest consensus state: current_term: 3 leader_uuid: "9ed5e67dc759449b960c2c71ee039c42" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:55.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.487833 1750 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: This master's current role is: FOLLOWER
08:03:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.501710 1252 heartbeater.cc:344] Connected to a master server at 127.0.200.188:39611
08:03:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.504946 1699 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "83bed13f5bbd49e897e7ee2c1682840c" instance_seqno: 1761379430956632) as {username='slave'} at 127.0.200.129:49521; Asking this server to re-register.
08:03:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.506021 1252 heartbeater.cc:461] Registering TS with master...
08:03:55.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.508285 1699 ts_manager.cc:194] Registered new tserver with Master: 83bed13f5bbd49e897e7ee2c1682840c (127.0.200.129:39163)
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.520882 1762 negotiation.cc:337] Failed RPC negotiation. Trace:
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.381351 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.0.200.190:34159 (local address 127.0.0.1:33238)
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.381949 (+ 598us) negotiation.cc:107] Waiting for socket to connect
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.381977 (+ 28us) client_negotiation.cc:174] Beginning negotiation
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.382129 (+ 152us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.484069 (+101940us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.484076 (+ 7us) client_negotiation.cc:394] Received NEGOTIATE response from server
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.484177 (+ 101us) client_negotiation.cc:189] Negotiated authn=SASL
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.484925 (+ 748us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.484938 (+ 13us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.519631 (+ 34693us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":391,"thread_start_us":263,"threads_started":1}
08:03:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.519454 1685 connection.cc:537] server connection from 127.0.0.1:46340 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:55.522 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:03:55.523 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.0.200.190:34159(127.0.200.190:34159)] handling channelInactive
08:03:55.523 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.190:34159(127.0.200.190:34159)] cleaning up while in state READY due to: connection closed
08:03:55.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.521509 1668 master_runner.cc:186] Error getting master registration for 127.0.200.190:34159: Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
08:03:55.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.526317 1668 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:55.623 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.190:34159(127.0.200.190:34159)] connecting to peer
08:03:55.624 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.624 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.190:34159(127.0.200.190:34159)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.624 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.0.200.190:34159(127.0.200.190:34159) for tablet Kudu Master: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.643 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.190:34159(127.0.200.190:34159)] connecting to peer
08:03:55.643 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.188:39611(127.0.200.188:39611)] connecting to peer
08:03:55.643 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.644 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.200.190:34159(127.0.200.190:34159)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.644 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.190:34159: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.644 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.0.200.188:39611(127.0.200.188:39611)] Successfully connected to peer
08:03:55.657 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x63b0c27b, L:/127.0.0.1:46344 - R:/127.0.200.188:39611] using TOKEN/null
08:03:55.659 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:55.663 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.190:34159(127.0.200.190:34159)] connecting to peer
08:03:55.663 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.663 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.200.190:34159(127.0.200.190:34159)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.664 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.190:34159: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.665 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:55.683 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.190:34159(127.0.200.190:34159)] connecting to peer
08:03:55.684 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.684 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.190:34159(127.0.200.190:34159)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.190:34159: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.685 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:55.703 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.190:34159(127.0.200.190:34159)] connecting to peer
08:03:55.703 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.704 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.200.190:34159(127.0.200.190:34159)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.704 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.190:34159: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.704 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:55.723 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.190:34159(127.0.200.190:34159)] connecting to peer
08:03:55.724 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.724 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.190:34159(127.0.200.190:34159)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.724 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.190:34159: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.725 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:55.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.752193 1773 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9ed5e67dc759449b960c2c71ee039c42)
08:03:55.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.752357 1773 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:55.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.752980 1773 leader_election.cc:290] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159), 4c116f66397249a491b1d9259e219594 (127.0.200.188:39611)
08:03:55.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.753767 912 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: connect: Connection refused (error 111) [suppressed 6 similar messages]
08:03:55.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.754885 912 leader_election.cc:336] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159): Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: connect: Connection refused (error 111)
08:03:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.759538 1709 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9809071a62ca479b8178425bef946094" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "4c116f66397249a491b1d9259e219594" is_pre_election: true
08:03:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.760202 911 leader_election.cc:304] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9809071a62ca479b8178425bef946094; no voters: 4c116f66397249a491b1d9259e219594, 9ed5e67dc759449b960c2c71ee039c42
08:03:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.760423 1773 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
08:03:55.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.776419 1750 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 9ed5e67dc759449b960c2c71ee039c42)
08:03:55.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.776588 1750 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:55.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.778003 1750 leader_election.cc:290] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159), 9809071a62ca479b8178425bef946094 (127.0.200.189:44271)
08:03:55.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.778592 1686 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: connect: Connection refused (error 111)
08:03:55.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.780493 1686 leader_election.cc:336] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159): Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: connect: Connection refused (error 111)
08:03:55.783 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.190:34159(127.0.200.190:34159)] connecting to peer
08:03:55.784 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.784 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.200.190:34159(127.0.200.190:34159)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.784 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.190:34159: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.784549 934 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c116f66397249a491b1d9259e219594" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "9809071a62ca479b8178425bef946094" is_pre_election: true
08:03:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.784758 934 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4c116f66397249a491b1d9259e219594 in term 3.
08:03:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.785609 1685 leader_election.cc:304] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4c116f66397249a491b1d9259e219594, 9809071a62ca479b8178425bef946094; no voters: 9ed5e67dc759449b960c2c71ee039c42
08:03:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.785944 1750 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 3 FOLLOWER]: Leader pre-election won for term 4
08:03:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.786041 1750 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 9ed5e67dc759449b960c2c71ee039c42)
08:03:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.786134 1750 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 3 FOLLOWER]: Advancing to term 4
08:03:55.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271; will retry
08:03:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.788929 1750 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.789570 1750 leader_election.cc:290] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [CANDIDATE]: Term 4 election: Requested vote from peers 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159), 9809071a62ca479b8178425bef946094 (127.0.200.189:44271)
08:03:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.790019 934 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c116f66397249a491b1d9259e219594" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "9809071a62ca479b8178425bef946094"
08:03:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.790217 934 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 3 FOLLOWER]: Advancing to term 4
08:03:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.790637 1686 leader_election.cc:336] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159): Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: connect: Connection refused (error 111)
08:03:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.792167 934 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4c116f66397249a491b1d9259e219594 in term 4.
08:03:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.792507 1685 leader_election.cc:304] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4c116f66397249a491b1d9259e219594, 9809071a62ca479b8178425bef946094; no voters: 9ed5e67dc759449b960c2c71ee039c42
08:03:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.792737 1750 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 4 FOLLOWER]: Leader election won for term 4
08:03:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.792862 1750 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [term 4 LEADER]: Becoming Leader. State: Replica: 4c116f66397249a491b1d9259e219594, State: Running, Role: LEADER
08:03:55.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.793264 1750 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } }
08:03:55.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.794378 1770 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4c116f66397249a491b1d9259e219594. Latest consensus state: current_term: 4 leader_uuid: "4c116f66397249a491b1d9259e219594" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:55.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.794503 1770 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: This master's current role is: LEADER
08:03:55.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.794922 1780 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:03:55.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.797044 1780 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067]
08:03:55.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.797418 1780 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=732382f1fe234432b85505b59925bd05]
08:03:55.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.798914 1780 tablet_loader.cc:96] loaded metadata for tablet 2be90dfcb5d04b93be51236b1d024232 (table txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067])
08:03:55.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.799263 1780 tablet_loader.cc:96] loaded metadata for tablet 39069dd33ea449b28c95ff20e283e6d7 (table kudu_system.kudu_transactions [id=732382f1fe234432b85505b59925bd05])
08:03:55.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.799589 1780 tablet_loader.cc:96] loaded metadata for tablet 687bc4b309484420ad10e1badd1ecd9e (table txn_manager_fallback_rolling [id=25a8a9f9357844238befa20212db8067])
08:03:55.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.799836 1780 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:03:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.800540 1780 catalog_manager.cc:1269] Loaded cluster ID: 11103c47897d43f1bc346107e55096ee
08:03:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.800612 1780 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:03:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.801481 1780 catalog_manager.cc:1514] Loading token signing keys...
08:03:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.802126 1780 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594: Loaded TSK: 0
08:03:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.802736 1780 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:03:55.843 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.0.200.190:34159(127.0.200.190:34159)] connecting to peer
08:03:55.844 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.844 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.190:34159(127.0.200.190:34159)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.845 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.200.190:34159: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159
08:03:55.845 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:55.846 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.0.200.190:34159, [1ms] received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [21ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [25ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [36ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [36ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [40ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [41ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [61ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [62ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [81ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [81ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [101ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [102ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [161ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [162ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [222ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [222ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, deferred=Deferred@1020030975(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1771505555@50096f72 -> (continuation of Deferred@2025114852 after retry RPC after error@1968803577) -> (continuation of Deferred@708162618 after retry RPC after error@828128497) -> (continuation of Deferred@1875969490 after retry RPC after error@1455841431) -> (continuation of Deferred@1372953105 after retry RPC after error@29265774) -> (continuation of Deferred@1722213662 after retry RPC after error@50455429) -> (continuation of Deferred@2121917670 after retry RPC after error@189383096), errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/263344142@5a165bda -> (continuation of Deferred@2025114852 after retry RPC after error@1968803577) -> (continuation of Deferred@708162618 after retry RPC after error@828128497) -> (continuation of Deferred@1875969490 after retry RPC after error@1455841431) -> (continuation of Deferred@1372953105 after retry RPC after error@29265774) -> (continuation of Deferred@1722213662 after retry RPC after error@50455429) -> (continuation of Deferred@2121917670 after retry RPC after error@189383096))) after lookup
08:03:55.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.874163 1729 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:46344 (request call id 7) took 27 ms (client timeout 27 ms). Trace:
08:03:55.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.874305 1729 rpcz_store.cc:269] 1025 08:03:55.846957 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:03:55.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.847020 (+ 63us) service_pool.cc:225] Handling call
08:03:55.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.874109 (+ 27089us) inbound_call.cc:173] Queueing success response
08:03:55.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":447,"init.queue_time_us":50,"thread_start_us":834,"threads_started":4}
08:03:55.875 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.0.200.190:34159, [1ms] received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [21ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [25ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [36ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [36ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [40ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [41ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [61ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [62ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [81ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [81ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [101ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [102ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [161ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [162ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [222ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [222ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [223ms] sending RPC to server master-127.0.200.188:39611, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1020030975(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1771505555@50096f72 -> (continuation of Deferred@2025114852 after retry RPC after error@1968803577) -> (continuation of Deferred@708162618 after retry RPC after error@828128497) -> (continuation of Deferred@1875969490 after retry RPC after error@1455841431) -> (continuation of Deferred@1372953105 after retry RPC after error@29265774) -> (continuation of Deferred@1722213662 after retry RPC after error@50455429) -> (continuation of Deferred@2121917670 after retry RPC after error@189383096) -> (continuation of Deferred@1451726550 after retry RPC@1743263284), errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/263344142@5a165bda -> (continuation of Deferred@2025114852 after retry RPC after error@1968803577) -> (continuation of Deferred@708162618 after retry RPC after error@828128497) -> (continuation of Deferred@1875969490 after retry RPC after error@1455841431) -> (continuation of Deferred@1372953105 after retry RPC after error@29265774) -> (continuation of Deferred@1722213662 after retry RPC after error@50455429) -> (continuation of Deferred@2121917670 after retry RPC after error@189383096) -> (continuation of Deferred@1451726550 after retry RPC@1743263284)))
08:03:55.876 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.0.200.190:34159, [1ms] received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [21ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [25ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [36ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [36ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [40ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [41ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [61ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [62ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [81ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [81ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [101ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [102ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [161ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [162ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.0.200.190:34159,127.0.200.188:39611,127.0.200.189:44271) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.190:34159, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.188:39611, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.200.189:44271, [222ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.190:34159: Network error: Failed to connect to peer master-127.0.200.190:34159(127.0.200.190:34159): Connection refused: /127.0.200.190:34159, [222ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.189:44271: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.0.200.188:39611: OK, [223ms] sending RPC to server master-127.0.200.188:39611, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1020030975(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1771505555@50096f72 -> (continuation of Deferred@2025114852 after retry RPC after error@1968803577) -> (continuation of Deferred@708162618 after retry RPC after error@828128497) -> (continuation of Deferred@1875969490 after retry RPC after error@1455841431) -> (continuation of Deferred@1372953105 after retry RPC after error@29265774) -> (continuation of Deferred@1722213662 after retry RPC after error@50455429) -> (continuation of Deferred@2121917670 after retry RPC after error@189383096) -> (continuation of Deferred@1451726550 after retry RPC@1743263284), errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/263344142@5a165bda -> (continuation of Deferred@2025114852 after retry RPC after error@1968803577) -> (continuation of Deferred@708162618 after retry RPC after error@828128497) -> (continuation of Deferred@1875969490 after retry RPC after error@1455841431) -> (continuation of Deferred@1372953105 after retry RPC after error@29265774) -> (continuation of Deferred@1722213662 after retry RPC after error@50455429) -> (continuation of Deferred@2121917670 after retry RPC after error@189383096) -> (continuation of Deferred@1451726550 after retry RPC@1743263284)))
08:03:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.876575 1699 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46362:
08:03:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:03:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:03:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:03:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:03:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:03:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:03:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.879588 1744 master.cc:464] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 1 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
08:03:55.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.885237 934 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [term 4 FOLLOWER]: Refusing update from remote peer 4c116f66397249a491b1d9259e219594: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
08:03:55.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.885778 1770 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
08:03:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.887331 1773 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4c116f66397249a491b1d9259e219594. Latest consensus state: current_term: 4 leader_uuid: "4c116f66397249a491b1d9259e219594" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.887603 1773 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: This master's current role is: FOLLOWER
08:03:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.888729 1750 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "4c116f66397249a491b1d9259e219594" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.888938 1750 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 [sys.catalog]: This master's current role is: LEADER
08:03:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:55.889003 1686 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 -> Peer 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159): Couldn't send request to peer 9ed5e67dc759449b960c2c71ee039c42. Status: Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.889786 1773 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "4c116f66397249a491b1d9259e219594" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9ed5e67dc759449b960c2c71ee039c42" member_type: VOTER last_known_addr { host: "127.0.200.190" port: 34159 } } peers { permanent_uuid: "9809071a62ca479b8178425bef946094" member_type: VOTER last_known_addr { host: "127.0.200.189" port: 44271 } } peers { permanent_uuid: "4c116f66397249a491b1d9259e219594" member_type: VOTER last_known_addr { host: "127.0.200.188" port: 39611 } } }
08:03:55.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.889952 1773 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9809071a62ca479b8178425bef946094 [sys.catalog]: This master's current role is: FOLLOWER
08:03:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.902653 1534 heartbeater.cc:344] Connected to a master server at 127.0.200.188:39611
08:03:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.903578 1699 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c0c510dd21c40039e7d4798a2c8a019" instance_seqno: 1761379431901111) as {username='slave'} at 127.0.200.131:54469; Asking this server to re-register.
08:03:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.904021 1534 heartbeater.cc:461] Registering TS with master...
08:03:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.904187 1534 heartbeater.cc:507] Master 127.0.200.188:39611 requested a full tablet report, sending...
08:03:55.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:55.905171 1699 ts_manager.cc:194] Registered new tserver with Master: 2c0c510dd21c40039e7d4798a2c8a019 (127.0.200.131:46651)
08:03:56.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.134346 1729 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:46344 (request call id 8) took 250 ms (client timeout 250 ms). Trace:
08:03:56.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.134474 1729 rpcz_store.cc:269] 1025 08:03:55.884169 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:03:56.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:55.884288 (+ 119us) service_pool.cc:225] Handling call
08:03:56.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:56.134277 (+249989us) inbound_call.cc:173] Queueing success response
08:03:56.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
08:03:56.135 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.0.200.188:39611, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@585719664(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1771505555@4a49187c, errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/263344142@201206c5))
08:03:56.135 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.0.200.188:39611, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@585719664(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1771505555@4a49187c, errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/263344142@201206c5))
08:03:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:56.139091 1278 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.027s from last keepalive heartbeat (effective timeout is 1.000s)
08:03:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.152168 1796 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
08:03:56.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.357641 1686 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 -> Peer 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159): Couldn't send request to peer 9ed5e67dc759449b960c2c71ee039c42. Status: Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:03:56.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:56.379004 1699 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9c8b20a175c74159881815bc68197aa4" instance_seqno: 1761379431444323) as {username='slave'} at 127.0.200.130:51261; Asking this server to re-register.
08:03:56.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:56.379601 1393 heartbeater.cc:461] Registering TS with master...
08:03:56.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:56.379791 1393 heartbeater.cc:507] Master 127.0.200.188:39611 requested a full tablet report, sending...
08:03:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:56.380975 1699 ts_manager.cc:194] Registered new tserver with Master: 9c8b20a175c74159881815bc68197aa4 (127.0.200.130:41469)
08:03:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.393507 1729 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:46344 (request call id 9) took 250 ms (client timeout 250 ms). Trace:
08:03:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.393641 1729 rpcz_store.cc:269] 1025 08:03:56.143229 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:03:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:56.143296 (+ 67us) service_pool.cc:225] Handling call
08:03:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:56.393433 (+250137us) inbound_call.cc:173] Queueing success response
08:03:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
08:03:56.394 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.0.200.188:39611, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@124579141(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1771505555@6f162007, errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/263344142@6d9e5f61))
08:03:56.394 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.0.200.188:39611, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@124579141(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1771505555@6f162007, errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/263344142@6d9e5f61))
08:03:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.653810 1729 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:46344 (request call id 10) took 250 ms (client timeout 250 ms). Trace:
08:03:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.653918 1729 rpcz_store.cc:269] 1025 08:03:56.403475 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:03:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:56.403554 (+ 79us) service_pool.cc:225] Handling call
08:03:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:56.653750 (+250196us) inbound_call.cc:173] Queueing success response
08:03:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
08:03:56.654 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.0.200.188:39611, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1560959033(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1771505555@3d6c8018, errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/263344142@63a499aa))
08:03:56.654 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.0.200.188:39611, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1560959033(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1771505555@3d6c8018, errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/263344142@63a499aa))
08:03:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.837337 1686 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 -> Peer 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159): Couldn't send request to peer 9ed5e67dc759449b960c2c71ee039c42. Status: Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:56.893123 1699 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46372:
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:03:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.893747 1699 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:56.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.906893 1729 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:46344 (request call id 11) took 242 ms (client timeout 250 ms). Trace:
08:03:56.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:56.906983 1729 rpcz_store.cc:269] 1025 08:03:56.664416 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:03:56.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:56.664679 (+ 263us) service_pool.cc:225] Handling call
08:03:56.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1025 08:03:56.906841 (+242162us) inbound_call.cc:173] Queueing success response
08:03:56.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
08:03:56.907 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 9c8b20a175c74159881815bc68197aa4 (127.0.200.130:41469): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1761379432 last_transition_timestamp: 1761379436
08:03:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:57.275321 1686 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 -> Peer 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159): Couldn't send request to peer 9ed5e67dc759449b960c2c71ee039c42. Status: Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
08:03:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.468531 1252 heartbeater.cc:499] Master 127.0.200.188:39611 was elected leader, sending a full tablet report...
08:03:57.526 [DEBUG - main] (Connection.java:575) [peer master-127.0.200.188:39611(127.0.200.188:39611)] disconnecting while in state READY
08:03:57.526 [DEBUG - main] (Connection.java:575) [peer master-127.0.200.190:34159(127.0.200.190:34159)] disconnecting while in state TERMINATED
08:03:57.527 [DEBUG - main] (Connection.java:575) [peer master-127.0.200.189:44271(127.0.200.189:44271)] disconnecting while in state READY
08:03:57.527 [DEBUG - main] (Connection.java:575) [peer 83bed13f5bbd49e897e7ee2c1682840c(127.0.200.129:39163)] disconnecting while in state READY
08:03:57.527 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 83bed13f5bbd49e897e7ee2c1682840c(127.0.200.129:39163)] handling channelInactive
08:03:57.527 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 83bed13f5bbd49e897e7ee2c1682840c(127.0.200.129:39163)] cleaning up while in state READY due to: connection closed
08:03:57.528 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:03:57.528 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.0.200.188:39611(127.0.200.188:39611)] handling channelInactive
08:03:57.528 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.188:39611(127.0.200.188:39611)] cleaning up while in state READY due to: connection closed
08:03:57.528 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.0.200.189:44271(127.0.200.189:44271)] handling channelInactive
08:03:57.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.528831 802 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu pid 1095
08:03:57.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.200.189:44271(127.0.200.189:44271)] cleaning up while in state READY due to: connection closed
08:03:57.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.535266 1247 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:57.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.623508 802 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu with pid 1095
08:03:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.641042 802 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu pid 1256
08:03:57.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.647097 1388 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:57.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.758370 802 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu with pid 1256
08:03:57.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:57.777446 1414 connection.cc:537] server connection from 127.0.200.130:41771 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:57.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.778052 802 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu pid 1398
08:03:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.784082 1529 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:57.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251025 08:03:57.790035 1686 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4c116f66397249a491b1d9259e219594 -> Peer 9ed5e67dc759449b960c2c71ee039c42 (127.0.200.190:34159): Couldn't send request to peer 9ed5e67dc759449b960c2c71ee039c42. Status: Network error: Client connection negotiation failed: client connection to 127.0.200.190:34159: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
08:03:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.873983 802 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu with pid 1398
08:03:57.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.891459 802 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu pid 891
08:03:57.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.892406 965 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:57.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:57.994284 802 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu with pid 891
08:03:58.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:58.013856 802 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu pid 1668
08:03:58.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:58.019680 1739 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:58.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251025 08:03:58.122395 802 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWhNLRi/build/asan/bin/kudu with pid 1668
08:03:58.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-25T08:03:58Z chronyd exiting
08:03:58.268 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)