08:04:27.816 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:04:27.816 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskV73fp_/test-tmp
08:04:27.817 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
08:04:27.817 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskV73fp_/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
08:04:27.818 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:04:28.130 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:04:28.130 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:04:28.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.132664 21428 mini_oidc.cc:150] Starting JWKS server
08:04:28.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.134718 21428 webserver.cc:492] Webserver started at http://127.0.0.1:35353/ using document root <none> and password file <none>
08:04:28.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.134929 21428 mini_oidc.cc:186] Starting OIDC Discovery server
08:04:28.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.135378 21428 webserver.cc:492] Webserver started at http://127.0.0.1:46721/ using document root <none> and password file <none>
08:04:28.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T08:04:28Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:04:28.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T08:04:28Z Disabled control of system clock
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.164158 21428 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/wal
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/data
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/logs
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/data/info.pb
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.62:46031
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.62
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35353/jwks.json
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:04:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:04:28.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.477434 21439 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:04:28.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.477735 21439 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:28.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.477814 21439 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:04:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.485780 21439 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:04:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.485879 21439 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:04:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.485929 21439 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:04:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.485970 21439 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.486029 21439 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:04:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.486070 21439 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.486112 21439 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35353/jwks.json
08:04:28.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.495647 21439 master_runner.cc:386] Master server non-default flags:
08:04:28.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:28.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:28.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:28.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:28.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/data
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/wal
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.62:46031
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35353/jwks.json
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/data/info.pb
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.62
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/logs
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:22 UTC on 24a791456cd2
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8676
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:04:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.497182 21439 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.498878 21439 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.506100 21444 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.506158 21445 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.506738 21447 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.507336 21439 server_base.cc:1047] running on GCE node
08:04:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.508050 21439 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:28.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.509313 21439 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:28.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.510509 21439 hybrid_clock.cc:648] HybridClock initialized: now 1761206668510447 us; error 51 us; skew 500 ppm
08:04:28.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.510778 21439 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:28.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.512861 21439 webserver.cc:492] Webserver started at http://127.20.237.62:46005/ using document root <none> and password file <none>
08:04:28.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.513482 21439 fs_manager.cc:362] Metadata directory not provided
08:04:28.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.513578 21439 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:28.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.513846 21439 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:04:28.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.516098 21439 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/data/instance:
08:04:28.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95c6cbdc462b459d91cf07e56273c510"
08:04:28.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:28 on dist-test-slave-zcf4"
08:04:28.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.516804 21439 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/wal/instance:
08:04:28.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95c6cbdc462b459d91cf07e56273c510"
08:04:28.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:28 on dist-test-slave-zcf4"
08:04:28.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.521554 21439 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.005s sys 0.001s
08:04:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.524528 21453 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.525681 21439 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.525833 21439 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/data,/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/wal
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95c6cbdc462b459d91cf07e56273c510"
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:28 on dist-test-slave-zcf4"
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.526008 21439 fs_report.cc:389] FS layout report
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/wal
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/wal
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/data/data
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:28.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.541128 21439 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:28.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.541893 21439 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:28.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.542189 21439 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:28.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.564535 21439 rpc_server.cc:307] RPC server started. Bound to: 127.20.237.62:46031
08:04:28.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.564574 21514 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.237.62:46031 every 8 connection(s)
08:04:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.566044 21439 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/data/info.pb
08:04:28.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.569103 21428 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV73fp_/build/asan/bin/kudu as pid 21439
08:04:28.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.569434 21428 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-0/wal/instance
08:04:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.571086 21515 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.572407 21428 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/logs
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data/info.pb
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.61:45365
08:04:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.61
08:04:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35353/jwks.json
08:04:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:04:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:04:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.579008 21515 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } has no permanent_uuid. Determining permanent_uuid...
08:04:28.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.595332 21515 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } has no permanent_uuid. Determining permanent_uuid...
08:04:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.596356 21457 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111)
08:04:28.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.597821 21515 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.61:45365: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111)
08:04:28.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.647265 21515 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } attempt: 1
08:04:28.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.649116 21515 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.61:45365: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111)
08:04:28.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.717447 21515 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } attempt: 2
08:04:28.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.719293 21515 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.61:45365: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111)
08:04:28.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.810602 21515 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } attempt: 3
08:04:28.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.812443 21515 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.61:45365: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111)
08:04:28.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.887562 21517 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:04:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.887892 21517 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.888026 21517 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:04:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.895905 21517 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:04:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.896008 21517 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:04:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.896085 21517 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:04:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.896142 21517 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.896231 21517 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:04:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.896296 21517 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.896363 21517 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35353/jwks.json
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.905985 21517 master_runner.cc:386] Master server non-default flags:
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.61:45365
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35353/jwks.json
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data/info.pb
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.61
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/logs
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:22 UTC on 24a791456cd2
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8676
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:04:28.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.907867 21517 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:28.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.909554 21517 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.917062 21526 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.917164 21525 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.917774 21528 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.918467 21517 server_base.cc:1047] running on GCE node
08:04:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.919198 21517 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.921448 21517 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.922662 21517 hybrid_clock.cc:648] HybridClock initialized: now 1761206668922608 us; error 27 us; skew 500 ppm
08:04:28.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.923005 21517 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:28.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.925036 21517 webserver.cc:492] Webserver started at http://127.20.237.61:33677/ using document root <none> and password file <none>
08:04:28.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.925701 21517 fs_manager.cc:362] Metadata directory not provided
08:04:28.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.925825 21517 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.926189 21517 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:04:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.928434 21517 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data/instance:
08:04:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e"
08:04:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:28 on dist-test-slave-zcf4"
08:04:28.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.929185 21517 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal/instance:
08:04:28.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e"
08:04:28.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:28 on dist-test-slave-zcf4"
08:04:28.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.934248 21517 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
08:04:28.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.937368 21534 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.938525 21517 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
08:04:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.938685 21517 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data,/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal
08:04:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e"
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:28 on dist-test-slave-zcf4"
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.938905 21517 fs_report.cc:389] FS layout report
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data/data
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.955528 21517 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.955675 21515 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } attempt: 4
08:04:28.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.956552 21517 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.956857 21517 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:28.957535 21515 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.61:45365: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111)
08:04:28.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.978186 21517 rpc_server.cc:307] RPC server started. Bound to: 127.20.237.61:45365
08:04:28.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.978227 21595 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.237.61:45365 every 8 connection(s)
08:04:28.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.979786 21517 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data/info.pb
08:04:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.980602 21428 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV73fp_/build/asan/bin/kudu as pid 21517
08:04:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.980778 21428 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal/instance
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.982034 21428 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/logs
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data/info.pb
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.60:44459
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.60
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35353/jwks.json
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:04:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:04:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.985266 21596 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:28.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:28.991425 21596 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } has no permanent_uuid. Determining permanent_uuid...
08:04:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.004181 21596 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } has no permanent_uuid. Determining permanent_uuid...
08:04:29.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.013036 21596 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } has no permanent_uuid. Determining permanent_uuid...
08:04:29.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.014107 21537 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:29.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.015726 21596 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.60:44459: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:29.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.065079 21596 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } attempt: 1
08:04:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.066946 21596 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.60:44459: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:29.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.135187 21596 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } attempt: 2
08:04:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.137096 21596 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.60:44459: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:29.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.228368 21596 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } attempt: 3
08:04:29.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.230177 21596 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.60:44459: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.256801 21515 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } attempt: 5
08:04:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.262717 21515 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } has no permanent_uuid. Determining permanent_uuid...
08:04:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.264526 21515 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.60:44459: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.303804 21598 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:04:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.304147 21598 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.304231 21598 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.312316 21598 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.312413 21598 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.312463 21598 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.312503 21598 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.312554 21598 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.312594 21598 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.312636 21598 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35353/jwks.json
08:04:29.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.315719 21515 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } attempt: 1
08:04:29.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.317543 21515 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.60:44459: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.323206 21598 master_runner.cc:386] Master server non-default flags:
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.60:44459
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35353/jwks.json
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data/info.pb
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.60
08:04:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/logs
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:22 UTC on 24a791456cd2
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8676
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:04:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.324749 21598 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.326583 21598 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.334249 21607 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.334277 21606 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:29.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.334960 21609 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:29.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.335352 21598 server_base.cc:1047] running on GCE node
08:04:29.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.336053 21598 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.337399 21598 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:29.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.338604 21598 hybrid_clock.cc:648] HybridClock initialized: now 1761206669338534 us; error 48 us; skew 500 ppm
08:04:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.338948 21598 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.341048 21598 webserver.cc:492] Webserver started at http://127.20.237.60:36411/ using document root <none> and password file <none>
08:04:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.341696 21598 fs_manager.cc:362] Metadata directory not provided
08:04:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.341795 21598 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.342084 21598 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:04:29.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.344426 21598 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data/instance:
08:04:29.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "36591742c42a4a36906a28f1b8029ece"
08:04:29.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:29 on dist-test-slave-zcf4"
08:04:29.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.345160 21598 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal/instance:
08:04:29.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "36591742c42a4a36906a28f1b8029ece"
08:04:29.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:29 on dist-test-slave-zcf4"
08:04:29.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.350250 21598 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
08:04:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.353412 21615 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:29.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.354605 21598 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
08:04:29.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.354730 21598 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data,/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal
08:04:29.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "36591742c42a4a36906a28f1b8029ece"
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:29 on dist-test-slave-zcf4"
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.354943 21598 fs_report.cc:389] FS layout report
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data/data
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.371577 21598 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.372364 21598 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.372602 21598 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.373448 21596 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } attempt: 4
08:04:29.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.375385 21596 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.60:44459: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:29.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.385756 21515 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } attempt: 2
08:04:29.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.387943 21515 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.237.60:44459: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:29.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.395004 21598 rpc_server.cc:307] RPC server started. Bound to: 127.20.237.60:44459
08:04:29.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.395051 21676 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.237.60:44459 every 8 connection(s)
08:04:29.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.396412 21598 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data/info.pb
08:04:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.399596 21428 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV73fp_/build/asan/bin/kudu as pid 21598
08:04:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.399750 21428 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal/instance
08:04:29.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.400517 21677 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:29.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.407773 21677 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } has no permanent_uuid. Determining permanent_uuid...
08:04:29.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.421036 21677 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } has no permanent_uuid. Determining permanent_uuid...
08:04:29.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.427645 21677 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } has no permanent_uuid. Determining permanent_uuid...
08:04:29.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.439416 21677 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: Bootstrap starting.
08:04:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.442634 21677 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: Neither blocks nor log segments found. Creating new log.
08:04:29.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.443656 21677 log.cc:826] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: Log is configured to *not* fsync() on all Append() calls
08:04:29.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.445937 21677 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: No bootstrap required, opened a new log
08:04:29.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.451013 21677 raft_consensus.cc:359] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:29.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.451287 21677 raft_consensus.cc:385] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:29.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.451393 21677 raft_consensus.cc:740] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 36591742c42a4a36906a28f1b8029ece, State: Initialized, Role: FOLLOWER
08:04:29.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.451931 21677 consensus_queue.cc:260] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:29.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.452831 21685 sys_catalog.cc:455] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:29.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.453155 21685 sys_catalog.cc:458] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: This master's current role is: FOLLOWER
08:04:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.453948 21677 sys_catalog.cc:565] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: configured and running, proceeding with master startup.
08:04:29.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.462785 21696 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:04:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.462921 21696 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:04:29.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.460124 21598 master_runner.cc:186] Error getting master registration for 127.20.237.62:46031: OK, instance_id { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" instance_seqno: 1761206668559443 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:04:29.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.465126 21598 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:04:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.494159 21515 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } attempt: 3
08:04:29.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.505856 21515 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510: Bootstrap starting.
08:04:29.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.510028 21515 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510: Neither blocks nor log segments found. Creating new log.
08:04:29.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.511417 21515 log.cc:826] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510: Log is configured to *not* fsync() on all Append() calls
08:04:29.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.514336 21515 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510: No bootstrap required, opened a new log
08:04:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.521826 21515 raft_consensus.cc:359] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.522140 21515 raft_consensus.cc:385] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.522259 21515 raft_consensus.cc:740] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95c6cbdc462b459d91cf07e56273c510, State: Initialized, Role: FOLLOWER
08:04:29.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.522961 21515 consensus_queue.cc:260] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:29.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.523795 21699 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:29.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.524027 21699 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: This master's current role is: FOLLOWER
08:04:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.525148 21515 sys_catalog.cc:565] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: configured and running, proceeding with master startup.
08:04:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.525789 21685 raft_consensus.cc:493] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:29.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.526024 21685 raft_consensus.cc:515] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:29.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.527774 21685 leader_election.cc:290] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031), 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365)
08:04:29.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.529119 21480 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "36591742c42a4a36906a28f1b8029ece" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95c6cbdc462b459d91cf07e56273c510" is_pre_election: true
08:04:29.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.529708 21480 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 36591742c42a4a36906a28f1b8029ece in term 0.
08:04:29.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.530305 21617 leader_election.cc:304] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 36591742c42a4a36906a28f1b8029ece, 95c6cbdc462b459d91cf07e56273c510; no voters:
08:04:29.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.530740 21685 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 0 FOLLOWER]: Leader pre-election won for term 1
08:04:29.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.530901 21685 raft_consensus.cc:493] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:04:29.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.531019 21685 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 0 FOLLOWER]: Advancing to term 1
08:04:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.534404 21685 raft_consensus.cc:515] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:29.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.536940 21480 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "36591742c42a4a36906a28f1b8029ece" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95c6cbdc462b459d91cf07e56273c510"
08:04:29.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.537134 21480 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 0 FOLLOWER]: Advancing to term 1
08:04:29.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.540517 21480 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 36591742c42a4a36906a28f1b8029ece in term 1.
08:04:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.540894 21617 leader_election.cc:304] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 36591742c42a4a36906a28f1b8029ece, 95c6cbdc462b459d91cf07e56273c510; no voters:
08:04:29.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.541775 21685 leader_election.cc:290] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [CANDIDATE]: Term 1 election: Requested vote from peers 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031), 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365)
08:04:29.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.542271 21685 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 1 FOLLOWER]: Leader election won for term 1
08:04:29.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.542510 21685 raft_consensus.cc:697] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 1 LEADER]: Becoming Leader. State: Replica: 36591742c42a4a36906a28f1b8029ece, State: Running, Role: LEADER
08:04:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.543788 21685 consensus_queue.cc:237] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:29.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:29.547988 21459 tablet.cc:2378] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:04:29.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.550424 21709 sys_catalog.cc:455] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: SysCatalogTable state changed. Reason: New leader 36591742c42a4a36906a28f1b8029ece. Latest consensus state: current_term: 1 leader_uuid: "36591742c42a4a36906a28f1b8029ece" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:29.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.550896 21709 sys_catalog.cc:458] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: This master's current role is: LEADER
08:04:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.556035 21712 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:04:29.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.564674 21713 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:04:29.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.560089 21439 master_runner.cc:186] Error getting master registration for 127.20.237.61:45365: OK, instance_id { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" instance_seqno: 1761206668972902 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:04:29.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.566725 21713 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:04:29.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.566830 21439 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:04:29.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.573845 21480 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 1 FOLLOWER]: Refusing update from remote peer 36591742c42a4a36906a28f1b8029ece: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:04:29.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.574829 21709 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [LEADER]: Connected to new peer: Peer: permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:04:29.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.585829 21699 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 36591742c42a4a36906a28f1b8029ece. Latest consensus state: current_term: 1 leader_uuid: "36591742c42a4a36906a28f1b8029ece" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:29.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.586074 21699 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: This master's current role is: FOLLOWER
08:04:29.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.588822 21685 sys_catalog.cc:455] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "36591742c42a4a36906a28f1b8029ece" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:29.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.589087 21685 sys_catalog.cc:458] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: This master's current role is: LEADER
08:04:29.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.590751 21713 catalog_manager.cc:1357] Generated new cluster ID: 40948aa4facf4848a044328dc5740242
08:04:29.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.590874 21713 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:04:29.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.591395 21699 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "36591742c42a4a36906a28f1b8029ece" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:29.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.591599 21699 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: This master's current role is: FOLLOWER
08:04:29.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.593350 21715 mvcc.cc:204] Tried to move back new op lower bound from 7213902518565154816 to 7213902518462164992. Current Snapshot: MvccSnapshot[applied={T|T < 7213902518565154816}]
08:04:29.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.606066 21713 catalog_manager.cc:1380] Generated new certificate authority record
08:04:29.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.608364 21713 catalog_manager.cc:1514] Loading token signing keys...
08:04:29.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.631624 21713 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: Generated new TSK 0
08:04:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.633395 21713 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.654244 21428 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/wal
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/data
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/logs
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/data/info.pb
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.1:0
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.237.1
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.1
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:04:29.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.674630 21596 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } attempt: 5
08:04:29.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.687791 21596 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: Bootstrap starting.
08:04:29.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.691949 21596 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: Neither blocks nor log segments found. Creating new log.
08:04:29.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.693490 21596 log.cc:826] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: Log is configured to *not* fsync() on all Append() calls
08:04:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.698014 21596 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: No bootstrap required, opened a new log
08:04:29.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.711488 21596 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:29.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.712317 21596 raft_consensus.cc:385] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:29.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.712710 21596 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1bdb4a0427c4400aaa3d458e9a0ec44e, State: Initialized, Role: FOLLOWER
08:04:29.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.714087 21596 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.715955 21721 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.716450 21721 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: This master's current role is: FOLLOWER
08:04:29.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.718465 21561 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "36591742c42a4a36906a28f1b8029ece" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" is_pre_election: true
08:04:29.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.718500 21560 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "36591742c42a4a36906a28f1b8029ece" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e"
08:04:29.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.719020 21561 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 36591742c42a4a36906a28f1b8029ece in term 0.
08:04:29.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.719650 21559 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 0 FOLLOWER]: Advancing to term 1
08:04:29.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.723403 21559 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 1 FOLLOWER]: Refusing update from remote peer 36591742c42a4a36906a28f1b8029ece: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:04:29.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.723928 21596 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: configured and running, proceeding with master startup.
08:04:29.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.724444 21709 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [LEADER]: Connected to new peer: Peer: permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:04:29.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.756600 21721 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 36591742c42a4a36906a28f1b8029ece. Latest consensus state: current_term: 1 leader_uuid: "36591742c42a4a36906a28f1b8029ece" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.756865 21721 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: This master's current role is: FOLLOWER
08:04:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.756685 21725 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "36591742c42a4a36906a28f1b8029ece" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.757438 21725 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: This master's current role is: FOLLOWER
08:04:29.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.773741 21737 catalog_manager.cc:1269] Loaded cluster ID: 40948aa4facf4848a044328dc5740242
08:04:29.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.774035 21737 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: loading cluster ID for follower catalog manager: success
08:04:29.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.777892 21737 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: acquiring CA information for follower catalog manager: success
08:04:29.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.780332 21737 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:29.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.843369 21709 sys_catalog.cc:455] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "36591742c42a4a36906a28f1b8029ece" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:29.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:29.843650 21709 sys_catalog.cc:458] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: This master's current role is: LEADER
08:04:30.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.079910 21719 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:04:30.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.080231 21719 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:30.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.080379 21719 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:04:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.088343 21719 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:04:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.088511 21719 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:04:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.088583 21719 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:04:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.088635 21719 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:04:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.088760 21719 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.237.1
08:04:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.098421 21719 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/data
08:04:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/wal
08:04:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.1:0
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/data/info.pb
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.1
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.237.1
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/logs
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:22 UTC on 24a791456cd2
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8676
08:04:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:04:30.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.100323 21719 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:30.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.102032 21719 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.111284 21743 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.111430 21744 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.112892 21746 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:30.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.113354 21719 server_base.cc:1047] running on GCE node
08:04:30.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.113898 21719 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:30.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.114928 21719 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:30.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.116151 21719 hybrid_clock.cc:648] HybridClock initialized: now 1761206670116088 us; error 46 us; skew 500 ppm
08:04:30.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.116411 21719 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:30.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.118620 21719 webserver.cc:492] Webserver started at http://127.20.237.1:41043/ using document root <none> and password file <none>
08:04:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.119324 21719 fs_manager.cc:362] Metadata directory not provided
08:04:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.119428 21719 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.119715 21719 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:04:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.122009 21719 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/data/instance:
08:04:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1b897c93d74495e9cb6f845c7fb09a8"
08:04:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:30 on dist-test-slave-zcf4"
08:04:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.122732 21719 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/wal/instance:
08:04:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1b897c93d74495e9cb6f845c7fb09a8"
08:04:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:30 on dist-test-slave-zcf4"
08:04:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.128615 21719 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.000s sys 0.005s
08:04:30.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.132980 21752 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:30.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.134614 21719 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.001s
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.134825 21719 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/data,/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/wal
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1b897c93d74495e9cb6f845c7fb09a8"
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:30 on dist-test-slave-zcf4"
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.135066 21719 fs_report.cc:389] FS layout report
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/wal
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/wal
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/data/data
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.161470 21719 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.162607 21719 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:30.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.163005 21719 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:30.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.166885 21719 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:04:30.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.167162 21719 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:04:30.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.167356 21719 ts_tablet_manager.cc:616] Registered 0 tablets
08:04:30.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.167527 21719 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:04:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.271059 21719 rpc_server.cc:307] RPC server started. Bound to: 127.20.237.1:34165
08:04:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.271077 21876 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.237.1:34165 every 8 connection(s)
08:04:30.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.272449 21719 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/data/info.pb
08:04:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.273669 21428 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV73fp_/build/asan/bin/kudu as pid 21719
08:04:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.273849 21428 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-0/wal/instance
08:04:30.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.276952 21878 heartbeater.cc:344] Connected to a master server at 127.20.237.62:46031
08:04:30.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.277418 21878 heartbeater.cc:461] Registering TS with master...
08:04:30.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.278728 21878 heartbeater.cc:507] Master 127.20.237.62:46031 requested a full tablet report, sending...
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.282171 21428 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/wal
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/data
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/logs
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/data/info.pb
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.2:0
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.237.2
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.2
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:04:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:04:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:04:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.285142 21469 ts_manager.cc:194] Registered new tserver with Master: a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165)
08:04:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.289191 21879 heartbeater.cc:344] Connected to a master server at 127.20.237.61:45365
08:04:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.289465 21879 heartbeater.cc:461] Registering TS with master...
08:04:30.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.290498 21879 heartbeater.cc:507] Master 127.20.237.61:45365 requested a full tablet report, sending...
08:04:30.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.292534 21551 ts_manager.cc:194] Registered new tserver with Master: a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165)
08:04:30.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.298822 21877 heartbeater.cc:344] Connected to a master server at 127.20.237.60:44459
08:04:30.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.299006 21877 heartbeater.cc:461] Registering TS with master...
08:04:30.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.299619 21877 heartbeater.cc:507] Master 127.20.237.60:44459 requested a full tablet report, sending...
08:04:30.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.301548 21632 ts_manager.cc:194] Registered new tserver with Master: a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165)
08:04:30.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.303761 21632 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.237.1:50409
08:04:30.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.561342 21712 catalog_manager.cc:1269] Loaded cluster ID: 40948aa4facf4848a044328dc5740242
08:04:30.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.561493 21712 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510: loading cluster ID for follower catalog manager: success
08:04:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.565523 21712 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510: acquiring CA information for follower catalog manager: success
08:04:30.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.568419 21712 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.625010 21881 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:04:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.625313 21881 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.625427 21881 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:04:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.634117 21881 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:04:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.634250 21881 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:04:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.634300 21881 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:04:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.634337 21881 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:04:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.634418 21881 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.237.2
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.644371 21881 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/data
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/wal
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.2:0
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/data/info.pb
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.2
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.237.2
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/logs
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:22 UTC on 24a791456cd2
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8676
08:04:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:04:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.646001 21881 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:30.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.647791 21881 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.657265 21888 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.658880 21887 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:30.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:30.660485 21890 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.661064 21881 server_base.cc:1047] running on GCE node
08:04:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.661705 21881 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:30.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.662648 21881 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.663882 21881 hybrid_clock.cc:648] HybridClock initialized: now 1761206670663803 us; error 53 us; skew 500 ppm
08:04:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.664222 21881 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.666602 21881 webserver.cc:492] Webserver started at http://127.20.237.2:35277/ using document root <none> and password file <none>
08:04:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.667382 21881 fs_manager.cc:362] Metadata directory not provided
08:04:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.667483 21881 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.667765 21881 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:04:30.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.670099 21881 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/data/instance:
08:04:30.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90e5f9f2bb4d4a80b42c34aba660acbb"
08:04:30.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:30 on dist-test-slave-zcf4"
08:04:30.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.670997 21881 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/wal/instance:
08:04:30.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90e5f9f2bb4d4a80b42c34aba660acbb"
08:04:30.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:30 on dist-test-slave-zcf4"
08:04:30.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.676512 21881 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.001s sys 0.003s
08:04:30.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.679975 21896 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.681376 21881 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
08:04:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.681519 21881 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/data,/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/wal
08:04:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90e5f9f2bb4d4a80b42c34aba660acbb"
08:04:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:30 on dist-test-slave-zcf4"
08:04:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.681686 21881 fs_report.cc:389] FS layout report
08:04:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/wal
08:04:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/wal
08:04:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/data/data
08:04:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:30.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.696791 21881 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:30.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.697563 21881 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:30.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.697806 21881 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.701529 21881 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:04:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.701800 21881 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
08:04:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.701925 21881 ts_tablet_manager.cc:616] Registered 0 tablets
08:04:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.701996 21881 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:04:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.804936 21881 rpc_server.cc:307] RPC server started. Bound to: 127.20.237.2:43461
08:04:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.804957 22017 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.237.2:43461 every 8 connection(s)
08:04:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.806284 21881 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/data/info.pb
08:04:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.807235 21428 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV73fp_/build/asan/bin/kudu as pid 21881
08:04:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.807466 21428 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-1/wal/instance
08:04:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.815382 21428 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/wal
08:04:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/data
08:04:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/logs
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/data/info.pb
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.3:0
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.237.3
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.3
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.816408 22018 heartbeater.cc:344] Connected to a master server at 127.20.237.60:44459
08:04:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.816742 22018 heartbeater.cc:461] Registering TS with master...
08:04:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.817746 22018 heartbeater.cc:507] Master 127.20.237.60:44459 requested a full tablet report, sending...
08:04:30.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.819339 21632 ts_manager.cc:194] Registered new tserver with Master: 90e5f9f2bb4d4a80b42c34aba660acbb (127.20.237.2:43461)
08:04:30.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.820353 21632 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.237.2:35183
08:04:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.828114 22019 heartbeater.cc:344] Connected to a master server at 127.20.237.62:46031
08:04:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.828372 22019 heartbeater.cc:461] Registering TS with master...
08:04:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.828881 22019 heartbeater.cc:507] Master 127.20.237.62:46031 requested a full tablet report, sending...
08:04:30.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.830313 21469 ts_manager.cc:194] Registered new tserver with Master: 90e5f9f2bb4d4a80b42c34aba660acbb (127.20.237.2:43461)
08:04:30.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.834087 22021 heartbeater.cc:344] Connected to a master server at 127.20.237.61:45365
08:04:30.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.834235 22021 heartbeater.cc:461] Registering TS with master...
08:04:30.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.834517 22021 heartbeater.cc:507] Master 127.20.237.61:45365 requested a full tablet report, sending...
08:04:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:30.835422 21551 ts_manager.cc:194] Registered new tserver with Master: 90e5f9f2bb4d4a80b42c34aba660acbb (127.20.237.2:43461)
08:04:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.173166 22020 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:04:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.173476 22020 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.173605 22020 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:04:31.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.182173 22020 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:04:31.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.182307 22020 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:04:31.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.182358 22020 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:04:31.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.182394 22020 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:04:31.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.182472 22020 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.237.3
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.192137 22020 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/data
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/wal
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.3:0
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/data/info.pb
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.3
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.237.3
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/logs
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:04:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:31.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:22 UTC on 24a791456cd2
08:04:31.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8676
08:04:31.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:04:31.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.193674 22020 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:31.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.195468 22020 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.204772 22028 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.204932 22027 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.205651 22030 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.207192 22020 server_base.cc:1047] running on GCE node
08:04:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.207693 22020 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:31.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.208549 22020 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.209759 22020 hybrid_clock.cc:648] HybridClock initialized: now 1761206671209706 us; error 38 us; skew 500 ppm
08:04:31.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.210005 22020 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:31.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.212158 22020 webserver.cc:492] Webserver started at http://127.20.237.3:40385/ using document root <none> and password file <none>
08:04:31.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.212873 22020 fs_manager.cc:362] Metadata directory not provided
08:04:31.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.212977 22020 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:31.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.213277 22020 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:04:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.215638 22020 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/data/instance:
08:04:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c"
08:04:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:31 on dist-test-slave-zcf4"
08:04:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.216333 22020 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/wal/instance:
08:04:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c"
08:04:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:31 on dist-test-slave-zcf4"
08:04:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.221271 22020 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.001s sys 0.004s
08:04:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.224303 22036 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.225577 22020 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
08:04:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.225708 22020 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/data,/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/wal
08:04:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c"
08:04:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:31 on dist-test-slave-zcf4"
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.225867 22020 fs_report.cc:389] FS layout report
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/wal
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/wal
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/data/data
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.239135 22020 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:31.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.239879 22020 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:31.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.240131 22020 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.243150 22020 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:04:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.243273 22020 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:04:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.243389 22020 ts_tablet_manager.cc:616] Registered 0 tablets
08:04:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.243474 22020 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:04:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.306752 21877 heartbeater.cc:499] Master 127.20.237.60:44459 was elected leader, sending a full tablet report...
08:04:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.316258 22020 rpc_server.cc:307] RPC server started. Bound to: 127.20.237.3:37041
08:04:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.316326 22157 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.237.3:37041 every 8 connection(s)
08:04:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.317651 22020 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/data/info.pb
08:04:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.321358 22159 heartbeater.cc:344] Connected to a master server at 127.20.237.62:46031
08:04:31.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.321803 22159 heartbeater.cc:461] Registering TS with master...
08:04:31.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.322716 22159 heartbeater.cc:507] Master 127.20.237.62:46031 requested a full tablet report, sending...
08:04:31.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.324541 21469 ts_manager.cc:194] Registered new tserver with Master: 59f9188ecd9f41c5978c24cb0e7b8d0c (127.20.237.3:37041)
08:04:31.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.326380 21428 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV73fp_/build/asan/bin/kudu as pid 22020
08:04:31.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.326531 21428 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/ts-2/wal/instance
08:04:31.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.328336 22160 heartbeater.cc:344] Connected to a master server at 127.20.237.61:45365
08:04:31.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.328621 22160 heartbeater.cc:461] Registering TS with master...
08:04:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.329303 22160 heartbeater.cc:507] Master 127.20.237.61:45365 requested a full tablet report, sending...
08:04:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.329367 22158 heartbeater.cc:344] Connected to a master server at 127.20.237.60:44459
08:04:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.329571 22158 heartbeater.cc:461] Registering TS with master...
08:04:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.330081 22158 heartbeater.cc:507] Master 127.20.237.60:44459 requested a full tablet report, sending...
08:04:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.330654 21551 ts_manager.cc:194] Registered new tserver with Master: 59f9188ecd9f41c5978c24cb0e7b8d0c (127.20.237.3:37041)
08:04:31.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.331041 21632 ts_manager.cc:194] Registered new tserver with Master: 59f9188ecd9f41c5978c24cb0e7b8d0c (127.20.237.3:37041)
08:04:31.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.332101 21632 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.237.3:48399
08:04:31.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.335397 21428 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:04:31.337 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:04:31.337 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:04:31.338 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.20.237.62"
port: 46031
}
bound_http_address {
host: "127.20.237.62"
port: 46005
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.20.237.61"
port: 45365
}
bound_http_address {
host: "127.20.237.61"
port: 33677
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.20.237.60"
port: 44459
}
bound_http_address {
host: "127.20.237.60"
port: 36411
}
}
}
08:04:31.338 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:04:31.339 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.20.237.1"
port: 34165
}
bound_http_address {
host: "127.20.237.1"
port: 41043
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.20.237.2"
port: 43461
}
bound_http_address {
host: "127.20.237.2"
port: 35277
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.20.237.3"
port: 37041
}
bound_http_address {
host: "127.20.237.3"
port: 40385
}
}
}
08:04:31.339 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:04:31.341 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:04:31.342 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.237.62:46031(127.20.237.62:46031)] connecting to peer
08:04:31.343 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:31.344 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.237.60:44459(127.20.237.60:44459)] connecting to peer
08:04:31.344 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.237.62:46031(127.20.237.62:46031)] Successfully connected to peer
08:04:31.345 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.237.61:45365(127.20.237.61:45365)] Successfully connected to peer
08:04:31.347 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.237.62
08:04:31.347 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.237.61
08:04:31.350 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.237.60:44459(127.20.237.60:44459)] Successfully connected to peer
08:04:31.351 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.20.237.60
08:04:31.356 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa15c931e, L:/127.0.0.1:38288 - R:/127.20.237.61:45365] using SASL/PLAIN
08:04:31.364 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3ba44d8f, L:/127.0.0.1:50194 - R:/127.20.237.62:46031] using SASL/PLAIN
08:04:31.365 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xc1df0852, L:/127.0.0.1:57252 - R:/127.20.237.60:44459] using SASL/PLAIN
08:04:31.367 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:04:31.368 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=26), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [16ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, deferred=null) after lookup
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.369938 21632 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57252:
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
08:04:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.372354 21632 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:04:31.392 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 6e2ee7d767ea40a7b365416d1fb98b19
08:04:31.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.448740 21949 tablet_service.cc:1505] Processing CreateTablet for tablet 08b057f94ae64be0a3d445c579c18414 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:04:31.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.449100 22092 tablet_service.cc:1505] Processing CreateTablet for tablet 08b057f94ae64be0a3d445c579c18414 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:04:31.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.449386 21812 tablet_service.cc:1505] Processing CreateTablet for tablet 0e1c9874fda544e8a1305b9a274ee9e2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:04:31.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.449581 21811 tablet_service.cc:1505] Processing CreateTablet for tablet 08b057f94ae64be0a3d445c579c18414 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:04:31.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.450260 22093 tablet_service.cc:1505] Processing CreateTablet for tablet 0e1c9874fda544e8a1305b9a274ee9e2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:04:31.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.450485 21949 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 08b057f94ae64be0a3d445c579c18414. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:31.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.450623 21950 tablet_service.cc:1505] Processing CreateTablet for tablet 0e1c9874fda544e8a1305b9a274ee9e2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:04:31.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.450760 21811 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 08b057f94ae64be0a3d445c579c18414. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:31.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.450485 22092 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 08b057f94ae64be0a3d445c579c18414. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:31.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.451270 21950 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0e1c9874fda544e8a1305b9a274ee9e2. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.452857 21812 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0e1c9874fda544e8a1305b9a274ee9e2. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:31.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.453956 22093 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0e1c9874fda544e8a1305b9a274ee9e2. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.467116 22186 tablet_bootstrap.cc:492] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8: Bootstrap starting.
08:04:31.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.471063 22185 tablet_bootstrap.cc:492] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb: Bootstrap starting.
08:04:31.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.471386 22187 tablet_bootstrap.cc:492] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c: Bootstrap starting.
08:04:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.472179 22186 tablet_bootstrap.cc:654] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8: Neither blocks nor log segments found. Creating new log.
08:04:31.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.473474 22186 log.cc:826] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8: Log is configured to *not* fsync() on all Append() calls
08:04:31.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.474392 22187 tablet_bootstrap.cc:654] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c: Neither blocks nor log segments found. Creating new log.
08:04:31.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.474920 22185 tablet_bootstrap.cc:654] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb: Neither blocks nor log segments found. Creating new log.
08:04:31.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.476069 22187 log.cc:826] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c: Log is configured to *not* fsync() on all Append() calls
08:04:31.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.476511 22185 log.cc:826] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb: Log is configured to *not* fsync() on all Append() calls
08:04:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.477903 22186 tablet_bootstrap.cc:492] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8: No bootstrap required, opened a new log
08:04:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.478196 22186 ts_tablet_manager.cc:1403] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.002s
08:04:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.478504 22187 tablet_bootstrap.cc:492] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c: No bootstrap required, opened a new log
08:04:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.478744 22187 ts_tablet_manager.cc:1403] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c: Time spent bootstrapping tablet: real 0.008s user 0.000s sys 0.005s
08:04:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.479234 22185 tablet_bootstrap.cc:492] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb: No bootstrap required, opened a new log
08:04:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.479501 22185 ts_tablet_manager.cc:1403] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.003s
08:04:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.483765 22187 raft_consensus.cc:359] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.484282 22187 raft_consensus.cc:385] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:31.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.484472 22187 raft_consensus.cc:740] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 59f9188ecd9f41c5978c24cb0e7b8d0c, State: Initialized, Role: FOLLOWER
08:04:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.485090 22187 consensus_queue.cc:260] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.485603 22186 raft_consensus.cc:359] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } }
08:04:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.486213 22158 heartbeater.cc:499] Master 127.20.237.60:44459 was elected leader, sending a full tablet report...
08:04:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.486550 22185 raft_consensus.cc:359] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.486888 22187 ts_tablet_manager.cc:1434] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c: Time spent starting tablet: real 0.008s user 0.005s sys 0.003s
08:04:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.486924 22185 raft_consensus.cc:385] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.487051 22185 raft_consensus.cc:740] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 90e5f9f2bb4d4a80b42c34aba660acbb, State: Initialized, Role: FOLLOWER
08:04:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.487422 22186 raft_consensus.cc:385] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.487605 22186 raft_consensus.cc:740] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a1b897c93d74495e9cb6f845c7fb09a8, State: Initialized, Role: FOLLOWER
08:04:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.488332 22186 consensus_queue.cc:260] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } }
08:04:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.488858 22187 tablet_bootstrap.cc:492] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c: Bootstrap starting.
08:04:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.491470 22187 tablet_bootstrap.cc:654] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c: Neither blocks nor log segments found. Creating new log.
08:04:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.491532 22185 consensus_queue.cc:260] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.499594 22186 ts_tablet_manager.cc:1434] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8: Time spent starting tablet: real 0.021s user 0.008s sys 0.012s
08:04:31.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.499686 22185 ts_tablet_manager.cc:1434] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb: Time spent starting tablet: real 0.020s user 0.011s sys 0.005s
08:04:31.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.499845 22187 tablet_bootstrap.cc:492] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c: No bootstrap required, opened a new log
08:04:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.499953 22018 heartbeater.cc:499] Master 127.20.237.60:44459 was elected leader, sending a full tablet report...
08:04:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.500088 22187 ts_tablet_manager.cc:1403] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.004s
08:04:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.500283 22186 tablet_bootstrap.cc:492] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8: Bootstrap starting.
08:04:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.500895 22187 raft_consensus.cc:359] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } }
08:04:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.501080 22187 raft_consensus.cc:385] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.501178 22187 raft_consensus.cc:740] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 59f9188ecd9f41c5978c24cb0e7b8d0c, State: Initialized, Role: FOLLOWER
08:04:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.501395 22187 consensus_queue.cc:260] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } }
08:04:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.502091 22186 tablet_bootstrap.cc:654] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8: Neither blocks nor log segments found. Creating new log.
08:04:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.502040 22187 ts_tablet_manager.cc:1434] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
08:04:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.506143 22185 tablet_bootstrap.cc:492] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb: Bootstrap starting.
08:04:31.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.508805 22185 tablet_bootstrap.cc:654] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb: Neither blocks nor log segments found. Creating new log.
08:04:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.517071 22185 tablet_bootstrap.cc:492] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb: No bootstrap required, opened a new log
08:04:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.517230 22185 ts_tablet_manager.cc:1403] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb: Time spent bootstrapping tablet: real 0.011s user 0.007s sys 0.003s
08:04:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.517424 22186 tablet_bootstrap.cc:492] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8: No bootstrap required, opened a new log
08:04:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.517587 22186 ts_tablet_manager.cc:1403] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8: Time spent bootstrapping tablet: real 0.017s user 0.005s sys 0.010s
08:04:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.517836 22185 raft_consensus.cc:359] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } }
08:04:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.517980 22185 raft_consensus.cc:385] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.518039 22185 raft_consensus.cc:740] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 90e5f9f2bb4d4a80b42c34aba660acbb, State: Initialized, Role: FOLLOWER
08:04:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.518220 22185 consensus_queue.cc:260] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } }
08:04:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.518371 22186 raft_consensus.cc:359] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.518537 22186 raft_consensus.cc:385] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.518620 22186 raft_consensus.cc:740] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a1b897c93d74495e9cb6f845c7fb09a8, State: Initialized, Role: FOLLOWER
08:04:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.518708 22185 ts_tablet_manager.cc:1434] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
08:04:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.518867 22186 consensus_queue.cc:260] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.519515 22186 ts_tablet_manager.cc:1434] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
08:04:31.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.532882 21880 tablet.cc:2378] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:04:31.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.539233 22193 raft_consensus.cc:493] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:31.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.539451 22193 raft_consensus.cc:515] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } }
08:04:31.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.541090 22193 leader_election.cc:290] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 59f9188ecd9f41c5978c24cb0e7b8d0c (127.20.237.3:37041), a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165)
08:04:31.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.548494 21832 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "0e1c9874fda544e8a1305b9a274ee9e2" candidate_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" is_pre_election: true
08:04:31.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.548542 22113 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "0e1c9874fda544e8a1305b9a274ee9e2" candidate_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" is_pre_election: true
08:04:31.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.548941 21832 raft_consensus.cc:2468] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 90e5f9f2bb4d4a80b42c34aba660acbb in term 0.
08:04:31.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.548981 22113 raft_consensus.cc:2468] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 90e5f9f2bb4d4a80b42c34aba660acbb in term 0.
08:04:31.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.549592 21899 leader_election.cc:304] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 59f9188ecd9f41c5978c24cb0e7b8d0c, 90e5f9f2bb4d4a80b42c34aba660acbb, a1b897c93d74495e9cb6f845c7fb09a8; no voters:
08:04:31.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.549952 22193 raft_consensus.cc:2804] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Leader pre-election won for term 1
08:04:31.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.550079 22193 raft_consensus.cc:493] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:04:31.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.550151 22193 raft_consensus.cc:3060] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Advancing to term 1
08:04:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.552312 22193 raft_consensus.cc:515] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } }
08:04:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.553023 22193 leader_election.cc:290] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [CANDIDATE]: Term 1 election: Requested vote from peers 59f9188ecd9f41c5978c24cb0e7b8d0c (127.20.237.3:37041), a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165)
08:04:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.553399 22113 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "0e1c9874fda544e8a1305b9a274ee9e2" candidate_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c"
08:04:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.553447 21832 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "0e1c9874fda544e8a1305b9a274ee9e2" candidate_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1b897c93d74495e9cb6f845c7fb09a8"
08:04:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.553576 22113 raft_consensus.cc:3060] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Advancing to term 1
08:04:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.553617 21832 raft_consensus.cc:3060] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Advancing to term 1
08:04:31.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.555907 21832 raft_consensus.cc:2468] T 0e1c9874fda544e8a1305b9a274ee9e2 P a1b897c93d74495e9cb6f845c7fb09a8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 90e5f9f2bb4d4a80b42c34aba660acbb in term 1.
08:04:31.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.555912 22113 raft_consensus.cc:2468] T 0e1c9874fda544e8a1305b9a274ee9e2 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 90e5f9f2bb4d4a80b42c34aba660acbb in term 1.
08:04:31.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.556304 21898 leader_election.cc:304] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 90e5f9f2bb4d4a80b42c34aba660acbb, a1b897c93d74495e9cb6f845c7fb09a8; no voters:
08:04:31.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.556574 22193 raft_consensus.cc:2804] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 1 FOLLOWER]: Leader election won for term 1
08:04:31.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.557029 22193 raft_consensus.cc:697] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 1 LEADER]: Becoming Leader. State: Replica: 90e5f9f2bb4d4a80b42c34aba660acbb, State: Running, Role: LEADER
08:04:31.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.557551 22193 consensus_queue.cc:237] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } }
08:04:31.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.563097 21632 catalog_manager.cc:5649] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb reported cstate change: term changed from 0 to 1, leader changed from <none> to 90e5f9f2bb4d4a80b42c34aba660acbb (127.20.237.2). New cstate: current_term: 1 leader_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } health_report { overall_health: UNKNOWN } } }
08:04:31.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.572106 22161 tablet.cc:2378] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:04:31.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.587428 22022 tablet.cc:2378] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:04:31.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.680508 22191 raft_consensus.cc:493] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:31.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.680728 22191 raft_consensus.cc:515] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.682461 22191 leader_election.cc:290] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165), 90e5f9f2bb4d4a80b42c34aba660acbb (127.20.237.2:43461)
08:04:31.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.688663 21832 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "08b057f94ae64be0a3d445c579c18414" candidate_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" is_pre_election: true
08:04:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.688642 21973 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "08b057f94ae64be0a3d445c579c18414" candidate_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" is_pre_election: true
08:04:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.688874 21832 raft_consensus.cc:2468] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59f9188ecd9f41c5978c24cb0e7b8d0c in term 0.
08:04:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.689020 21973 raft_consensus.cc:2468] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59f9188ecd9f41c5978c24cb0e7b8d0c in term 0.
08:04:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.689352 22038 leader_election.cc:304] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 59f9188ecd9f41c5978c24cb0e7b8d0c, a1b897c93d74495e9cb6f845c7fb09a8; no voters:
08:04:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.689723 22191 raft_consensus.cc:2804] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Leader pre-election won for term 1
08:04:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.689857 22191 raft_consensus.cc:493] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:04:31.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.689949 22191 raft_consensus.cc:3060] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Advancing to term 1
08:04:31.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.691958 22191 raft_consensus.cc:515] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.692507 22191 leader_election.cc:290] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [CANDIDATE]: Term 1 election: Requested vote from peers a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165), 90e5f9f2bb4d4a80b42c34aba660acbb (127.20.237.2:43461)
08:04:31.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.692950 21832 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "08b057f94ae64be0a3d445c579c18414" candidate_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1b897c93d74495e9cb6f845c7fb09a8"
08:04:31.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.693013 21973 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "08b057f94ae64be0a3d445c579c18414" candidate_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb"
08:04:31.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.693176 21832 raft_consensus.cc:3060] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Advancing to term 1
08:04:31.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.693179 21973 raft_consensus.cc:3060] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Advancing to term 1
08:04:31.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.696278 21832 raft_consensus.cc:2468] T 08b057f94ae64be0a3d445c579c18414 P a1b897c93d74495e9cb6f845c7fb09a8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59f9188ecd9f41c5978c24cb0e7b8d0c in term 1.
08:04:31.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.696273 21973 raft_consensus.cc:2468] T 08b057f94ae64be0a3d445c579c18414 P 90e5f9f2bb4d4a80b42c34aba660acbb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59f9188ecd9f41c5978c24cb0e7b8d0c in term 1.
08:04:31.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.696759 22039 leader_election.cc:304] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 59f9188ecd9f41c5978c24cb0e7b8d0c, 90e5f9f2bb4d4a80b42c34aba660acbb; no voters:
08:04:31.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.697069 22191 raft_consensus.cc:2804] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 1 FOLLOWER]: Leader election won for term 1
08:04:31.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.697542 22191 raft_consensus.cc:697] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 1 LEADER]: Becoming Leader. State: Replica: 59f9188ecd9f41c5978c24cb0e7b8d0c, State: Running, Role: LEADER
08:04:31.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.698276 22191 consensus_queue.cc:237] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.703505 21632 catalog_manager.cc:5649] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c reported cstate change: term changed from 0 to 1, leader changed from <none> to 59f9188ecd9f41c5978c24cb0e7b8d0c (127.20.237.3). New cstate: current_term: 1 leader_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } health_report { overall_health: HEALTHY } } }
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.763360 21632 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57260:
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:04:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:31.764307 21632 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:04:31.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.776731 21812 tablet_service.cc:1505] Processing CreateTablet for tablet 39a9dffb524f44f6acbe35914817b65e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=5fd10891efd149429f1c33147a56eb32]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:04:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.777024 21949 tablet_service.cc:1505] Processing CreateTablet for tablet 39a9dffb524f44f6acbe35914817b65e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=5fd10891efd149429f1c33147a56eb32]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:04:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.777209 22092 tablet_service.cc:1505] Processing CreateTablet for tablet 39a9dffb524f44f6acbe35914817b65e (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=5fd10891efd149429f1c33147a56eb32]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:04:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.777827 21949 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 39a9dffb524f44f6acbe35914817b65e. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.777829 21812 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 39a9dffb524f44f6acbe35914817b65e. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.778141 22092 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 39a9dffb524f44f6acbe35914817b65e. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:31.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.783437 22187 tablet_bootstrap.cc:492] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: Bootstrap starting.
08:04:31.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.784927 22187 tablet_bootstrap.cc:654] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: Neither blocks nor log segments found. Creating new log.
08:04:31.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.785678 22185 tablet_bootstrap.cc:492] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb: Bootstrap starting.
08:04:31.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.785904 22186 tablet_bootstrap.cc:492] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8: Bootstrap starting.
08:04:31.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.787446 22186 tablet_bootstrap.cc:654] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8: Neither blocks nor log segments found. Creating new log.
08:04:31.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.787637 22185 tablet_bootstrap.cc:654] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb: Neither blocks nor log segments found. Creating new log.
08:04:31.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.789280 22186 tablet_bootstrap.cc:492] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8: No bootstrap required, opened a new log
08:04:31.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.789443 22186 ts_tablet_manager.cc:1403] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
08:04:31.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.790017 22187 tablet_bootstrap.cc:492] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: No bootstrap required, opened a new log
08:04:31.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.790032 22186 raft_consensus.cc:359] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.790221 22186 raft_consensus.cc:385] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:31.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.790254 22187 ts_tablet_manager.cc:1403] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: Time spent bootstrapping tablet: real 0.007s user 0.004s sys 0.000s
08:04:31.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.790323 22186 raft_consensus.cc:740] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a1b897c93d74495e9cb6f845c7fb09a8, State: Initialized, Role: FOLLOWER
08:04:31.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.790552 22186 consensus_queue.cc:260] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.791006 22186 ts_tablet_manager.cc:1434] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8: Time spent starting tablet: real 0.001s user 0.001s sys 0.001s
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.791381 22192 tablet_replica.cc:442] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } }
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.791591 22192 tablet_replica.cc:445] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8: This TxnStatusTablet replica's current role is: FOLLOWER
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.791488 22187 raft_consensus.cc:359] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.791738 22187 raft_consensus.cc:385] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.791817 22187 raft_consensus.cc:740] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 59f9188ecd9f41c5978c24cb0e7b8d0c, State: Initialized, Role: FOLLOWER
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.792128 22187 consensus_queue.cc:260] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.793357 22191 tablet_replica.cc:442] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } }
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.793658 22191 tablet_replica.cc:445] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: This TxnStatusTablet replica's current role is: FOLLOWER
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.794195 22187 ts_tablet_manager.cc:1434] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: Time spent starting tablet: real 0.004s user 0.003s sys 0.000s
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.796823 22185 tablet_bootstrap.cc:492] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb: No bootstrap required, opened a new log
08:04:31.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.797004 22185 ts_tablet_manager.cc:1403] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb: Time spent bootstrapping tablet: real 0.011s user 0.008s sys 0.000s
08:04:31.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.798193 22185 raft_consensus.cc:359] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.798358 22185 raft_consensus.cc:385] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:31.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.798425 22185 raft_consensus.cc:740] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 90e5f9f2bb4d4a80b42c34aba660acbb, State: Initialized, Role: FOLLOWER
08:04:31.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.798705 22185 consensus_queue.cc:260] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:31.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.799394 22185 ts_tablet_manager.cc:1434] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
08:04:31.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.799345 22193 tablet_replica.cc:442] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } }
08:04:31.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:31.799645 22193 tablet_replica.cc:445] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb: This TxnStatusTablet replica's current role is: FOLLOWER
08:04:32.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.031697 22193 consensus_queue.cc:1048] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [LEADER]: Connected to new peer: Peer: permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:04:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.038141 22199 consensus_queue.cc:1048] T 0e1c9874fda544e8a1305b9a274ee9e2 P 90e5f9f2bb4d4a80b42c34aba660acbb [LEADER]: Connected to new peer: Peer: permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:04:32.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.099376 22191 consensus_queue.cc:1048] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [LEADER]: Connected to new peer: Peer: permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:04:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.105135 22201 raft_consensus.cc:493] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.105304 22201 raft_consensus.cc:515] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.105969 22201 leader_election.cc:290] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165), 90e5f9f2bb4d4a80b42c34aba660acbb (127.20.237.2:43461)
08:04:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.107400 21832 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "39a9dffb524f44f6acbe35914817b65e" candidate_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" is_pre_election: true
08:04:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.107620 21832 raft_consensus.cc:2468] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59f9188ecd9f41c5978c24cb0e7b8d0c in term 0.
08:04:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.107973 21973 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "39a9dffb524f44f6acbe35914817b65e" candidate_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" is_pre_election: true
08:04:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.106128 22201 consensus_queue.cc:1048] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c [LEADER]: Connected to new peer: Peer: permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.002s
08:04:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.108214 21973 raft_consensus.cc:2468] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59f9188ecd9f41c5978c24cb0e7b8d0c in term 0.
08:04:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.108042 22038 leader_election.cc:304] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 59f9188ecd9f41c5978c24cb0e7b8d0c, a1b897c93d74495e9cb6f845c7fb09a8; no voters:
08:04:32.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.108788 22201 raft_consensus.cc:2804] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Leader pre-election won for term 1
08:04:32.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.108892 22201 raft_consensus.cc:493] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:04:32.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.108996 22201 raft_consensus.cc:3060] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 0 FOLLOWER]: Advancing to term 1
08:04:32.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.112504 22201 raft_consensus.cc:515] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.113857 21832 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "39a9dffb524f44f6acbe35914817b65e" candidate_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a1b897c93d74495e9cb6f845c7fb09a8"
08:04:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.114035 21832 raft_consensus.cc:3060] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8 [term 0 FOLLOWER]: Advancing to term 1
08:04:32.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.114310 21973 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "39a9dffb524f44f6acbe35914817b65e" candidate_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb"
08:04:32.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.114516 21973 raft_consensus.cc:3060] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb [term 0 FOLLOWER]: Advancing to term 1
08:04:32.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.114754 22201 leader_election.cc:290] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [CANDIDATE]: Term 1 election: Requested vote from peers a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165), 90e5f9f2bb4d4a80b42c34aba660acbb (127.20.237.2:43461)
08:04:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.116219 21832 raft_consensus.cc:2468] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59f9188ecd9f41c5978c24cb0e7b8d0c in term 1.
08:04:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.116492 21973 raft_consensus.cc:2468] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59f9188ecd9f41c5978c24cb0e7b8d0c in term 1.
08:04:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.116643 22038 leader_election.cc:304] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 59f9188ecd9f41c5978c24cb0e7b8d0c, a1b897c93d74495e9cb6f845c7fb09a8; no voters:
08:04:32.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.116959 22201 raft_consensus.cc:2804] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 1 FOLLOWER]: Leader election won for term 1
08:04:32.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.117129 22201 raft_consensus.cc:697] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [term 1 LEADER]: Becoming Leader. State: Replica: 59f9188ecd9f41c5978c24cb0e7b8d0c, State: Running, Role: LEADER
08:04:32.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.117609 22201 consensus_queue.cc:237] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } }
08:04:32.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.118872 22191 tablet_replica.cc:442] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: TxnStatusTablet state changed. Reason: New leader 59f9188ecd9f41c5978c24cb0e7b8d0c. Latest consensus state: current_term: 1 leader_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } }
08:04:32.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.119096 22191 tablet_replica.cc:445] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: This TxnStatusTablet replica's current role is: LEADER
08:04:32.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.119832 22224 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
08:04:32.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.120213 22224 txn_status_manager.cc:930] Loading transaction status metadata into memory...
08:04:32.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.121106 21631 catalog_manager.cc:5649] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c reported cstate change: term changed from 0 to 1, leader changed from <none> to 59f9188ecd9f41c5978c24cb0e7b8d0c (127.20.237.3). New cstate: current_term: 1 leader_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } health_report { overall_health: HEALTHY } } }
08:04:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.166030 21832 raft_consensus.cc:1275] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8 [term 1 FOLLOWER]: Refusing update from remote peer 59f9188ecd9f41c5978c24cb0e7b8d0c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:04:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.166101 21973 raft_consensus.cc:1275] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb [term 1 FOLLOWER]: Refusing update from remote peer 59f9188ecd9f41c5978c24cb0e7b8d0c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:04:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.166671 22191 consensus_queue.cc:1048] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [LEADER]: Connected to new peer: Peer: permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:04:32.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.166950 22201 consensus_queue.cc:1048] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c [LEADER]: Connected to new peer: Peer: permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:04:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.175549 22192 tablet_replica.cc:442] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8: TxnStatusTablet state changed. Reason: New leader 59f9188ecd9f41c5978c24cb0e7b8d0c. Latest consensus state: current_term: 1 leader_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } }
08:04:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.175841 22192 tablet_replica.cc:445] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8: This TxnStatusTablet replica's current role is: FOLLOWER
08:04:32.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.182750 22219 mvcc.cc:204] Tried to move back new op lower bound from 7213902529185288192 to 7213902528995409920. Current Snapshot: MvccSnapshot[applied={T|T < 7213902529185288192}]
08:04:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.185829 22199 tablet_replica.cc:442] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb: TxnStatusTablet state changed. Reason: New leader 59f9188ecd9f41c5978c24cb0e7b8d0c. Latest consensus state: current_term: 1 leader_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } }
08:04:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.186175 22199 tablet_replica.cc:445] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb: This TxnStatusTablet replica's current role is: FOLLOWER
08:04:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.191569 22191 tablet_replica.cc:442] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } }
08:04:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.193164 22192 tablet_replica.cc:442] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } }
08:04:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.193373 22192 tablet_replica.cc:445] T 39a9dffb524f44f6acbe35914817b65e P a1b897c93d74495e9cb6f845c7fb09a8: This TxnStatusTablet replica's current role is: FOLLOWER
08:04:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.197199 22191 tablet_replica.cc:445] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: This TxnStatusTablet replica's current role is: LEADER
08:04:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.196519 22221 tablet_replica.cc:442] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } }
08:04:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.197762 22221 tablet_replica.cc:445] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c: This TxnStatusTablet replica's current role is: LEADER
08:04:32.202 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
08:04:32.204 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 6e2ee7d767ea40a7b365416d1fb98b19
08:04:32.204 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.237.60:44459
08:04:32.205 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:04:32.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.207670 22199 tablet_replica.cc:442] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" member_type: VOTER last_known_addr { host: "127.20.237.1" port: 34165 } } peers { permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" member_type: VOTER last_known_addr { host: "127.20.237.2" port: 43461 } } peers { permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" member_type: VOTER last_known_addr { host: "127.20.237.3" port: 37041 } } }
08:04:32.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.207875 22199 tablet_replica.cc:445] T 39a9dffb524f44f6acbe35914817b65e P 90e5f9f2bb4d4a80b42c34aba660acbb: This TxnStatusTablet replica's current role is: FOLLOWER
08:04:32.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.208925 21428 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV73fp_/build/asan/bin/kudu pid 21598
08:04:32.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.210409 21672 generic_service.cc:196] Checking for leaks (request via RPC)
08:04:32.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.341576 21428 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV73fp_/build/asan/bin/kudu with pid 21598
08:04:32.364 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.237.60:44459(127.20.237.60:44459)] handling channelInactive
08:04:32.364 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.237.60:44459(127.20.237.60:44459)] cleaning up while in state READY due to: connection closed
08:04:32.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:32.365118 21536 connection.cc:537] server connection from 127.0.0.1:38274 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:04:32.365 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:32.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.430822 22235 raft_consensus.cc:493] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 36591742c42a4a36906a28f1b8029ece)
08:04:32.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.431331 22235 raft_consensus.cc:515] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:32.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:32.431665 22018 heartbeater.cc:646] Failed to heartbeat to 127.20.237.60:44459 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:32.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.433007 22235 leader_election.cc:290] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365), 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459)
08:04:32.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.433426 21561 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "95c6cbdc462b459d91cf07e56273c510" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" is_pre_election: true
08:04:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:32.435727 21456 leader_election.cc:336] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459): Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.435846 21456 leader_election.cc:304] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 95c6cbdc462b459d91cf07e56273c510; no voters: 1bdb4a0427c4400aaa3d458e9a0ec44e, 36591742c42a4a36906a28f1b8029ece
08:04:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.436149 22235 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:04:32.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:32.561551 22158 heartbeater.cc:646] Failed to heartbeat to 127.20.237.60:44459 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:32.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.642902 22239 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 36591742c42a4a36906a28f1b8029ece)
08:04:32.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.643088 22239 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:32.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.644311 22239 leader_election.cc:290] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031), 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459)
08:04:32.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.644799 21480 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "95c6cbdc462b459d91cf07e56273c510" is_pre_election: true
08:04:32.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.645015 21480 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1bdb4a0427c4400aaa3d458e9a0ec44e in term 1.
08:04:32.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.645630 21536 leader_election.cc:304] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1bdb4a0427c4400aaa3d458e9a0ec44e, 95c6cbdc462b459d91cf07e56273c510; no voters:
08:04:32.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.646008 22239 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 1 FOLLOWER]: Leader pre-election won for term 2
08:04:32.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.646163 22239 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 1 FOLLOWER]: Starting leader election (detected failure of leader 36591742c42a4a36906a28f1b8029ece)
08:04:32.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.646276 22239 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 1 FOLLOWER]: Advancing to term 2
08:04:32.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:32.646421 21537 leader_election.cc:336] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459): Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:32.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.648339 22239 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:32.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.648869 22239 leader_election.cc:290] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 2 election: Requested vote from peers 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031), 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459)
08:04:32.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.649319 21480 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "95c6cbdc462b459d91cf07e56273c510"
08:04:32.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.649523 21480 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 1 FOLLOWER]: Advancing to term 2
08:04:32.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:32.650120 21537 leader_election.cc:336] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459): Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:32.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.651711 21480 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1bdb4a0427c4400aaa3d458e9a0ec44e in term 2.
08:04:32.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.652148 21536 leader_election.cc:304] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1bdb4a0427c4400aaa3d458e9a0ec44e, 95c6cbdc462b459d91cf07e56273c510; no voters: 36591742c42a4a36906a28f1b8029ece
08:04:32.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.652456 22239 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 2 FOLLOWER]: Leader election won for term 2
08:04:32.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.652786 22239 raft_consensus.cc:697] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 2 LEADER]: Becoming Leader. State: Replica: 1bdb4a0427c4400aaa3d458e9a0ec44e, State: Running, Role: LEADER
08:04:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.653154 22239 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.654434 22242 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1bdb4a0427c4400aaa3d458e9a0ec44e. Latest consensus state: current_term: 2 leader_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:32.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.654723 22242 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: This master's current role is: LEADER
08:04:32.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.655359 22243 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:04:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.657667 22243 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=5fd10891efd149429f1c33147a56eb32]
08:04:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.658195 22243 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19]
08:04:32.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.659953 22243 tablet_loader.cc:96] loaded metadata for tablet 08b057f94ae64be0a3d445c579c18414 (table txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19])
08:04:32.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.660391 22243 tablet_loader.cc:96] loaded metadata for tablet 0e1c9874fda544e8a1305b9a274ee9e2 (table txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19])
08:04:32.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.660727 22243 tablet_loader.cc:96] loaded metadata for tablet 39a9dffb524f44f6acbe35914817b65e (table kudu_system.kudu_transactions [id=5fd10891efd149429f1c33147a56eb32])
08:04:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.661031 22243 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:04:32.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.661996 22243 catalog_manager.cc:1269] Loaded cluster ID: 40948aa4facf4848a044328dc5740242
08:04:32.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.662073 22243 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:04:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.663215 22243 catalog_manager.cc:1514] Loading token signing keys...
08:04:32.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.663988 22243 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: Loaded TSK: 0
08:04:32.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.664752 22243 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:04:32.723 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.60:44459(127.20.237.60:44459)] connecting to peer
08:04:32.724 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.237.60:44459(127.20.237.60:44459): Connection refused: /127.20.237.60:44459
08:04:32.724 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.237.60:44459(127.20.237.60:44459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.60:44459(127.20.237.60:44459): Connection refused: /127.20.237.60:44459
08:04:32.724 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.20.237.60:44459(127.20.237.60:44459) for tablet Kudu Master: Failed to connect to peer master-127.20.237.60:44459(127.20.237.60:44459): Connection refused: /127.20.237.60:44459
08:04:32.743 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.60:44459(127.20.237.60:44459)] connecting to peer
08:04:32.744 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.237.60:44459(127.20.237.60:44459): Connection refused: /127.20.237.60:44459
08:04:32.744 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.237.60:44459(127.20.237.60:44459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.60:44459(127.20.237.60:44459): Connection refused: /127.20.237.60:44459
08:04:32.744 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.60:44459: Failed to connect to peer master-127.20.237.60:44459(127.20.237.60:44459): Connection refused: /127.20.237.60:44459
08:04:32.746 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:04:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.746528 21480 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 2 FOLLOWER]: Refusing update from remote peer 1bdb4a0427c4400aaa3d458e9a0ec44e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
08:04:32.746 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=24), Traces: [0ms] sending RPC to server master-127.20.237.60:44459, [2ms] received response from server master-127.20.237.60:44459: Network error: Failed to connect to peer master-127.20.237.60:44459(127.20.237.60:44459): Connection refused: /127.20.237.60:44459, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.20.237.60:44459(127.20.237.60:44459): Connection refused: /127.20.237.60:44459, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [22ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: Network error: Failed to connect to peer master-127.20.237.60:44459(127.20.237.60:44459): Connection refused: /127.20.237.60:44459, [22ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: OK, deferred=Deferred@1622281913(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@2c7adb89, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@31066655)) after lookup
08:04:32.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.747324 22242 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [LEADER]: Connected to new peer: Peer: permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
08:04:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.748783 22235 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1bdb4a0427c4400aaa3d458e9a0ec44e. Latest consensus state: current_term: 2 leader_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.749015 22235 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: This master's current role is: FOLLOWER
08:04:32.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.750656 22235 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:32.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.750865 22235 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: This master's current role is: FOLLOWER
08:04:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.764708 22242 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.764915 22242 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: This master's current role is: LEADER
08:04:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:32.765815 21537 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e -> Peer 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459): Couldn't send request to peer 36591742c42a4a36906a28f1b8029ece. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:32.787922 21550 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38298:
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:32.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:04:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:32.793114 21550 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:04:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:33.203518 21877 heartbeater.cc:646] Failed to heartbeat to 127.20.237.60:44459 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111)
08:04:33.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:33.235365 21537 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e -> Peer 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459): Couldn't send request to peer 36591742c42a4a36906a28f1b8029ece. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:04:33.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:33.724954 21537 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e -> Peer 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459): Couldn't send request to peer 36591742c42a4a36906a28f1b8029ece. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
08:04:34.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.044740 21537 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111) [suppressed 19 similar messages]
08:04:34.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.199577 21879 heartbeater.cc:499] Master 127.20.237.61:45365 was elected leader, sending a full tablet report...
08:04:34.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.282503 21537 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e -> Peer 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459): Couldn't send request to peer 36591742c42a4a36906a28f1b8029ece. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
08:04:34.370 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 0e1c9874fda544e8a1305b9a274ee9e2 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
08:04:34.370 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 08b057f94ae64be0a3d445c579c18414 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
08:04:34.370 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=0e1c9874fda544e8a1305b9a274ee9e2@[59f9188ecd9f41c5978c24cb0e7b8d0c(127.20.237.3:37041),90e5f9f2bb4d4a80b42c34aba660acbb(127.20.237.2:43461)[L],a1b897c93d74495e9cb6f845c7fb09a8(127.20.237.1:34165)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=08b057f94ae64be0a3d445c579c18414@[59f9188ecd9f41c5978c24cb0e7b8d0c(127.20.237.3:37041)[L],90e5f9f2bb4d4a80b42c34aba660acbb(127.20.237.2:43461),a1b897c93d74495e9cb6f845c7fb09a8(127.20.237.1:34165)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
08:04:34.370 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.237.61:45365, [1ms] Sub RPC GetTableLocations: received response from server master-127.20.237.61:45365: OK, deferred=null) after lookup
08:04:34.370 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90e5f9f2bb4d4a80b42c34aba660acbb(127.20.237.2:43461)] connecting to peer
08:04:34.371 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 90e5f9f2bb4d4a80b42c34aba660acbb(127.20.237.2:43461)] Successfully connected to peer
08:04:34.382 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa1450544, L:/127.0.0.1:60064 - R:/127.20.237.2:43461] using TOKEN/null
08:04:34.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.386380 21940 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.20.237.60:44459)
08:04:34.420 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.237.60:44459
08:04:34.420 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.421754 21428 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/logs
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data/info.pb
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.60:44459
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.60
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36411
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35353/jwks.json
08:04:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:04:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:04:34.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.432545 22021 heartbeater.cc:499] Master 127.20.237.61:45365 was elected leader, sending a full tablet report...
08:04:34.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.563717 22160 heartbeater.cc:499] Master 127.20.237.61:45365 was elected leader, sending a full tablet report...
08:04:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.575583 21439 master_runner.cc:186] Error getting master registration for 127.20.237.60:44459: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111),
08:04:34.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.575804 21439 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:04:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.754580 22285 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:04:34.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.754918 22285 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:34.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.755004 22285 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:04:34.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.758509 21537 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e -> Peer 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459): Couldn't send request to peer 36591742c42a4a36906a28f1b8029ece. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.60:44459: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
08:04:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.763375 22285 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:04:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.763463 22285 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:04:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.763510 22285 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:04:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.763550 22285 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.763603 22285 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:04:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.763649 22285 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.763700 22285 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35353/jwks.json
08:04:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.773567 22285 master_runner.cc:386] Master server non-default flags:
08:04:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.60:44459
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35353/jwks.json
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data/info.pb
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.60
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36411
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/logs
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:04:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:22 UTC on 24a791456cd2
08:04:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8676
08:04:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:04:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.775388 22285 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:34.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.777134 22285 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:34.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.785197 22293 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:34.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.785240 22294 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:34.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:34.786346 22296 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:34.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.786587 22285 server_base.cc:1047] running on GCE node
08:04:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.787405 22285 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:34.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.788760 22285 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:34.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.789963 22285 hybrid_clock.cc:648] HybridClock initialized: now 1761206674789893 us; error 52 us; skew 500 ppm
08:04:34.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.790313 22285 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:34.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.792569 22285 webserver.cc:492] Webserver started at http://127.20.237.60:36411/ using document root <none> and password file <none>
08:04:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.793300 22285 fs_manager.cc:362] Metadata directory not provided
08:04:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.793442 22285 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.798825 22285 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.002s sys 0.001s
08:04:34.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.801659 22302 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.803020 22285 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
08:04:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.803195 22285 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data,/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal
08:04:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "36591742c42a4a36906a28f1b8029ece"
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:29 on dist-test-slave-zcf4"
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.803910 22285 fs_report.cc:389] FS layout report
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/wal
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data/data
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:34.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.822685 22285 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:34.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.823630 22285 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:34.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.823912 22285 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.847563 22285 rpc_server.cc:307] RPC server started. Bound to: 127.20.237.60:44459
08:04:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.847581 22363 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.237.60:44459 every 8 connection(s)
08:04:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.849231 22285 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-2/data/info.pb
08:04:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.850986 21428 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV73fp_/build/asan/bin/kudu as pid 22285
08:04:34.851 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:34.851 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.237.61:45365
08:04:34.851 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:04:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.851804 21428 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV73fp_/build/asan/bin/kudu pid 21517
08:04:34.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.853008 21591 generic_service.cc:196] Checking for leaks (request via RPC)
08:04:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.858737 22364 sys_catalog.cc:263] Verifying existing consensus state
08:04:34.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.863907 22364 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: Bootstrap starting.
08:04:34.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.888332 22364 log.cc:826] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: Log is configured to *not* fsync() on all Append() calls
08:04:34.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.904989 22364 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:34.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.905624 22364 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: Bootstrap complete.
08:04:34.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.914525 22364 raft_consensus.cc:359] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:34.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.915565 22364 raft_consensus.cc:740] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 36591742c42a4a36906a28f1b8029ece, State: Initialized, Role: FOLLOWER
08:04:34.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.916287 22364 consensus_queue.cc:260] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:34.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.917102 22369 sys_catalog.cc:455] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:34.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.917349 22369 sys_catalog.cc:458] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: This master's current role is: FOLLOWER
08:04:34.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.918409 22364 sys_catalog.cc:565] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: configured and running, proceeding with master startup.
08:04:34.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.929333 22380 catalog_manager.cc:1269] Loaded cluster ID: 40948aa4facf4848a044328dc5740242
08:04:34.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.929682 22380 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: loading cluster ID for follower catalog manager: success
08:04:34.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.935351 22380 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: acquiring CA information for follower catalog manager: success
08:04:34.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:34.938104 22380 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.020467 21428 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV73fp_/build/asan/bin/kudu with pid 21517
08:04:35.043 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.237.61:45365(127.20.237.61:45365)] handling channelInactive
08:04:35.043 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:35.043 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state READY due to: connection closed
08:04:35.044 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.20.237.61:45365(127.20.237.61:45365) for tablet Kudu Master: connection closed
08:04:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:35.043721 22383 negotiation.cc:337] Failed RPC negotiation. Trace:
08:04:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.946690 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.20.237.61:45365 (local address 127.0.0.1:38312)
08:04:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.946877 (+ 187us) negotiation.cc:107] Waiting for socket to connect
08:04:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.946891 (+ 14us) client_negotiation.cc:174] Beginning negotiation
08:04:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.946977 (+ 86us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
08:04:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:35.042336 (+ 95359us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:04:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":56}
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:35.044394 22367 negotiation.cc:337] Failed RPC negotiation. Trace:
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.855069 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:57698 (local address 127.20.237.60:44459)
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.856346 (+ 1277us) server_negotiation.cc:207] Beginning negotiation
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.856405 (+ 59us) server_negotiation.cc:400] Waiting for connection header
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.856510 (+ 105us) server_negotiation.cc:408] Connection header received
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.858104 (+ 1594us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.858127 (+ 23us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.858672 (+ 545us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:34.859138 (+ 466us) server_negotiation.cc:227] Negotiated authn=SASL
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:35.020385 (+161247us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:35.022140 (+ 1755us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:35.042429 (+ 20289us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:57698: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":703,"thread_start_us":418,"threads_started":1}
08:04:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:35.045204 21454 connection.cc:537] server connection from 127.0.0.1:50164 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:04:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.045110 22285 master_runner.cc:186] Error getting master registration for 127.20.237.61:45365: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
08:04:35.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.048938 22285 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:04:35.063 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:35.064 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.60:44459(127.20.237.60:44459)] connecting to peer
08:04:35.064 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.237.60:44459(127.20.237.60:44459)] Successfully connected to peer
08:04:35.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.61:45365: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.082 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc3af817b, L:/127.0.0.1:57708 - R:/127.20.237.60:44459] using TOKEN/null
08:04:35.086 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459; will retry
08:04:35.102 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:35.103 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.104 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.104 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.61:45365: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.105 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459; will retry
08:04:35.122 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:35.124 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.124 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.124 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.61:45365: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.125 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459; will retry
08:04:35.142 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=4, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.20.237.61:45365, [161ms] received response from server master-127.20.237.61:45365: Network error: connection closed, [162ms] delaying RPC due to: Network error: connection closed, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [183ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: Network error: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [185ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [204ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [222ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: Network error: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [222ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [242ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: Network error: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [243ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, deferred=Deferred@1416830642(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@15a96cf3 -> (continuation of Deferred@187767974 after retry RPC after error@62729793) -> (continuation of Deferred@1333212682 after retry RPC after error@300357091) -> (continuation of Deferred@313589589 after retry RPC after error@2014647396), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@ea4ba0a -> (continuation of Deferred@187767974 after retry RPC after error@62729793) -> (continuation of Deferred@1333212682 after retry RPC after error@300357091) -> (continuation of Deferred@313589589 after retry RPC after error@2014647396)))
08:04:35.144 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
08:04:35.162 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:35.163 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.163 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.163 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.61:45365: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.164 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459; will retry
08:04:35.182 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:35.184 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.184 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.184 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.61:45365: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.185 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459; will retry
08:04:35.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.188727 22369 raft_consensus.cc:493] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:35.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.189033 22369 raft_consensus.cc:515] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:35.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.191561 22369 leader_election.cc:290] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031), 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365)
08:04:35.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.191978 21480 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "36591742c42a4a36906a28f1b8029ece" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "95c6cbdc462b459d91cf07e56273c510" is_pre_election: true
08:04:35.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:35.192301 22306 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111)
08:04:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:35.193938 22306 leader_election.cc:336] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365): Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111)
08:04:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.194129 22306 leader_election.cc:304] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 36591742c42a4a36906a28f1b8029ece; no voters: 1bdb4a0427c4400aaa3d458e9a0ec44e, 95c6cbdc462b459d91cf07e56273c510
08:04:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.194515 22369 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:04:35.202 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:35.204 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.204 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.204 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.61:45365: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.205 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459; will retry
08:04:35.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.212785 21877 heartbeater.cc:344] Connected to a master server at 127.20.237.60:44459
08:04:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.215013 22319 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" instance_seqno: 1761206670258861) as {username='slave'} at 127.20.237.1:58449; Asking this server to re-register.
08:04:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.215761 21877 heartbeater.cc:461] Registering TS with master...
08:04:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.217283 22319 ts_manager.cc:194] Registered new tserver with Master: a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165)
08:04:35.222 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:35.223 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.223 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.224 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.61:45365: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.225 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459; will retry
08:04:35.242 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:35.243 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.243 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.243 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.61:45365: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.245 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459; will retry
08:04:35.283 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:35.284 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.284 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.284 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.61:45365: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.285 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459; will retry
08:04:35.303 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:35.304 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.304 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.304 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.61:45365: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.305 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459; will retry
08:04:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.323525 22387 raft_consensus.cc:493] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 1bdb4a0427c4400aaa3d458e9a0ec44e)
08:04:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.323741 22387 raft_consensus.cc:515] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:35.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.330921 22387 leader_election.cc:290] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365), 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459)
08:04:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:35.335000 21457 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111) [suppressed 8 similar messages]
08:04:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:35.337894 21457 leader_election.cc:336] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365): Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111)
08:04:35.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.342599 22329 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "95c6cbdc462b459d91cf07e56273c510" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "36591742c42a4a36906a28f1b8029ece" is_pre_election: true
08:04:35.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.343137 22329 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 95c6cbdc462b459d91cf07e56273c510 in term 1.
08:04:35.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.343695 21456 leader_election.cc:304] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 36591742c42a4a36906a28f1b8029ece, 95c6cbdc462b459d91cf07e56273c510; no voters: 1bdb4a0427c4400aaa3d458e9a0ec44e
08:04:35.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.344009 22387 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 2 FOLLOWER]: Leader pre-election won for term 3
08:04:35.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.344141 22387 raft_consensus.cc:493] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 1bdb4a0427c4400aaa3d458e9a0ec44e)
08:04:35.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.344237 22387 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 2 FOLLOWER]: Advancing to term 3
08:04:35.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.347405 22387 raft_consensus.cc:515] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:35.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.348479 22387 leader_election.cc:290] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [CANDIDATE]: Term 3 election: Requested vote from peers 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365), 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459)
08:04:35.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.348594 22329 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "95c6cbdc462b459d91cf07e56273c510" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "36591742c42a4a36906a28f1b8029ece"
08:04:35.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.348811 22329 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 1 FOLLOWER]: Advancing to term 3
08:04:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:35.349498 21457 leader_election.cc:336] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365): Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111)
08:04:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.352795 22329 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 95c6cbdc462b459d91cf07e56273c510 in term 3.
08:04:35.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.353229 21456 leader_election.cc:304] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 36591742c42a4a36906a28f1b8029ece, 95c6cbdc462b459d91cf07e56273c510; no voters: 1bdb4a0427c4400aaa3d458e9a0ec44e
08:04:35.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.353561 22387 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 3 FOLLOWER]: Leader election won for term 3
08:04:35.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.354036 22387 raft_consensus.cc:697] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [term 3 LEADER]: Becoming Leader. State: Replica: 95c6cbdc462b459d91cf07e56273c510, State: Running, Role: LEADER
08:04:35.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.354501 22387 consensus_queue.cc:237] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:35.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.356398 22391 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 95c6cbdc462b459d91cf07e56273c510. Latest consensus state: current_term: 3 leader_uuid: "95c6cbdc462b459d91cf07e56273c510" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:35.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.356817 22391 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: This master's current role is: LEADER
08:04:35.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.357497 22393 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:04:35.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.359911 22393 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=5fd10891efd149429f1c33147a56eb32]
08:04:35.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.360461 22393 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19]
08:04:35.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.362066 22393 tablet_loader.cc:96] loaded metadata for tablet 08b057f94ae64be0a3d445c579c18414 (table txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19])
08:04:35.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.362427 22393 tablet_loader.cc:96] loaded metadata for tablet 0e1c9874fda544e8a1305b9a274ee9e2 (table txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19])
08:04:35.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.362691 22393 tablet_loader.cc:96] loaded metadata for tablet 39a9dffb524f44f6acbe35914817b65e (table kudu_system.kudu_transactions [id=5fd10891efd149429f1c33147a56eb32])
08:04:35.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.362958 22393 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:04:35.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.363698 22393 catalog_manager.cc:1269] Loaded cluster ID: 40948aa4facf4848a044328dc5740242
08:04:35.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.363801 22393 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:04:35.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.364830 22393 catalog_manager.cc:1514] Loading token signing keys...
08:04:35.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.365592 22393 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510: Loaded TSK: 0
08:04:35.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.366348 22393 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:04:35.423 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [1ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: Network error: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [2ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [22ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: Network error: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [23ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [42ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: Network error: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [42ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [62ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: Network error: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [62ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [81ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: Network error: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [82ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [122ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: Network error: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [123ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [142ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: Network error: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [142ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [142ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, deferred=Deferred@771554484(state=PENDING, result=null, callback=(continuation of Deferred@582274756 after retry RPC after error@146733055) -> (continuation of Deferred@291245555 after retry RPC after error@1536117296) -> (continuation of Deferred@1598651755 after retry RPC after error@548890738) -> (continuation of Deferred@1366673922 after retry RPC after error@1879753942) -> (continuation of Deferred@1681673625 after retry RPC after error@2136169876) -> (continuation of Deferred@634278 after retry RPC after error@637793541) -> (continuation of Deferred@542450905 after retry RPC after error@1158636218), errback=(continuation of Deferred@582274756 after retry RPC after error@146733055) -> (continuation of Deferred@291245555 after retry RPC after error@1536117296) -> (continuation of Deferred@1598651755 after retry RPC after error@548890738) -> (continuation of Deferred@1366673922 after retry RPC after error@1879753942) -> (continuation of Deferred@1681673625 after retry RPC after error@2136169876) -> (continuation of Deferred@634278 after retry RPC after error@637793541) -> (continuation of Deferred@542450905 after retry RPC after error@1158636218)))
08:04:35.423 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
08:04:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:35.432372 21457 consensus_peers.cc:597] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 -> Peer 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365): Couldn't send request to peer 1bdb4a0427c4400aaa3d458e9a0ec44e. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:35.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.441488 22018 heartbeater.cc:344] Connected to a master server at 127.20.237.60:44459
08:04:35.442 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:35.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.443188 22319 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" instance_seqno: 1761206670796356) as {username='slave'} at 127.20.237.2:37391; Asking this server to re-register.
08:04:35.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.443914 22018 heartbeater.cc:461] Registering TS with master...
08:04:35.444 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.444 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.61:45365: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365
08:04:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.445271 22319 ts_manager.cc:194] Registered new tserver with Master: 90e5f9f2bb4d4a80b42c34aba660acbb (127.20.237.2:43461)
08:04:35.446 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:04:35.446 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [3ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: Network error: Failed to connect to peer master-127.20.237.61:45365(127.20.237.61:45365): Connection refused: /127.20.237.61:45365, [3ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, deferred=null) after lookup
08:04:35.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.476518 22329 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 3 FOLLOWER]: Refusing update from remote peer 95c6cbdc462b459d91cf07e56273c510: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
08:04:35.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.477432 22391 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [LEADER]: Connected to new peer: Peer: permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
08:04:35.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.480638 22399 sys_catalog.cc:455] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "95c6cbdc462b459d91cf07e56273c510" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:35.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.480993 22399 sys_catalog.cc:458] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: This master's current role is: FOLLOWER
08:04:35.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.481530 22369 sys_catalog.cc:455] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: SysCatalogTable state changed. Reason: New leader 95c6cbdc462b459d91cf07e56273c510. Latest consensus state: current_term: 3 leader_uuid: "95c6cbdc462b459d91cf07e56273c510" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:35.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.481761 22369 sys_catalog.cc:458] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: This master's current role is: FOLLOWER
08:04:35.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.482409 22369 sys_catalog.cc:455] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "95c6cbdc462b459d91cf07e56273c510" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:35.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.482558 22369 sys_catalog.cc:458] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: This master's current role is: FOLLOWER
08:04:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.488219 22391 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "95c6cbdc462b459d91cf07e56273c510" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.488515 22391 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 [sys.catalog]: This master's current role is: LEADER
08:04:35.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.504465 21470 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50228:
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:04:35.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:35.506176 21470 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:04:35.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.577940 22158 heartbeater.cc:344] Connected to a master server at 127.20.237.60:44459
08:04:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.579295 22319 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" instance_seqno: 1761206671308578) as {username='slave'} at 127.20.237.3:48897; Asking this server to re-register.
08:04:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.579874 22158 heartbeater.cc:461] Registering TS with master...
08:04:35.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:35.580965 22319 ts_manager.cc:194] Registered new tserver with Master: 59f9188ecd9f41c5978c24cb0e7b8d0c (127.20.237.3:37041)
08:04:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:35.976071 21457 consensus_peers.cc:597] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 -> Peer 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365): Couldn't send request to peer 1bdb4a0427c4400aaa3d458e9a0ec44e. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:04:36.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:36.437507 22019 heartbeater.cc:499] Master 127.20.237.62:46031 was elected leader, sending a full tablet report...
08:04:36.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:36.495785 21457 consensus_peers.cc:597] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 -> Peer 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365): Couldn't send request to peer 1bdb4a0427c4400aaa3d458e9a0ec44e. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
08:04:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:36.571614 22159 heartbeater.cc:499] Master 127.20.237.62:46031 was elected leader, sending a full tablet report...
08:04:37.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.002211 21457 consensus_peers.cc:597] T 00000000000000000000000000000000 P 95c6cbdc462b459d91cf07e56273c510 -> Peer 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365): Couldn't send request to peer 1bdb4a0427c4400aaa3d458e9a0ec44e. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.61:45365: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
08:04:37.048 [DEBUG - Time-limited test] (Connection.java:748) [peer 59f9188ecd9f41c5978c24cb0e7b8d0c(127.20.237.3:37041)] connecting to peer
08:04:37.048 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 59f9188ecd9f41c5978c24cb0e7b8d0c(127.20.237.3:37041)] Successfully connected to peer
08:04:37.059 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x318b1315, L:/127.0.0.1:60418 - R:/127.20.237.3:37041] using TOKEN/null
08:04:37.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.068753 22067 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.20.237.60:44459)
08:04:37.098 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.237.61:45365
08:04:37.098 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.099460 21428 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV73fp_/build/asan/bin/kudu
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/logs
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data/info.pb
08:04:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.61:45365
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.61
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33677
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35353/jwks.json
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:04:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:04:37.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.206218 21878 heartbeater.cc:499] Master 127.20.237.62:46031 was elected leader, sending a full tablet report...
08:04:37.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.436658 22434 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:04:37.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.437002 22434 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:37.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.437121 22434 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:04:37.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.445708 22434 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:04:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.445814 22434 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:04:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.445894 22434 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:04:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.445948 22434 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.446025 22434 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:04:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.446089 22434 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.446161 22434 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35353/jwks.json
08:04:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.456202 22434 master_runner.cc:386] Master server non-default flags:
08:04:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.237.20:42973
08:04:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:04:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data
08:04:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.237.62:46031,127.20.237.61:45365,127.20.237.60:44459
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.237.61:45365
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35353/jwks.json
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data/info.pb
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.237.61
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33677
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/logs
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:22 UTC on 24a791456cd2
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8676
08:04:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:04:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.458153 22434 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.460026 22434 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:37.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.468106 22442 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:37.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.468151 22441 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:37.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.468900 22444 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:37.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.469368 22434 server_base.cc:1047] running on GCE node
08:04:37.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.470188 22434 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:37.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.471611 22434 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:37.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.472839 22434 hybrid_clock.cc:648] HybridClock initialized: now 1761206677472771 us; error 41 us; skew 500 ppm
08:04:37.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.473181 22434 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:37.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.475396 22434 webserver.cc:492] Webserver started at http://127.20.237.61:33677/ using document root <none> and password file <none>
08:04:37.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.476099 22434 fs_manager.cc:362] Metadata directory not provided
08:04:37.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.476243 22434 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:37.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.481354 22434 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
08:04:37.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.484099 22450 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:37.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.485298 22434 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
08:04:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.485450 22434 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data,/tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal
08:04:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e"
08:04:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:04:28 on dist-test-slave-zcf4"
08:04:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.486148 22434 fs_report.cc:389] FS layout report
08:04:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal
08:04:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/wal
08:04:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data/data
08:04:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:37.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.513677 22434 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:37.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.514585 22434 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:37.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.514910 22434 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.537209 22434 rpc_server.cc:307] RPC server started. Bound to: 127.20.237.61:45365
08:04:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.537230 22511 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.237.61:45365 every 8 connection(s)
08:04:37.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.538898 22434 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskV73fp_/test-tmp/mini-kudu-cluster7170280515268138055/master-1/data/info.pb
08:04:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.546957 21428 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskV73fp_/build/asan/bin/kudu as pid 22434
08:04:37.547 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:37.547 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.237.62:46031
08:04:37.547 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:04:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.547731 21428 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV73fp_/build/asan/bin/kudu pid 21439
08:04:37.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.548959 21510 generic_service.cc:196] Checking for leaks (request via RPC)
08:04:37.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.549172 22512 sys_catalog.cc:263] Verifying existing consensus state
08:04:37.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.553565 22512 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: Bootstrap starting.
08:04:37.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.577394 22512 log.cc:826] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: Log is configured to *not* fsync() on all Append() calls
08:04:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.602838 22160 heartbeater.cc:344] Connected to a master server at 127.20.237.61:45365
08:04:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.603720 22512 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.604331 22512 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: Bootstrap complete.
08:04:37.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.610456 22512 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:37.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.611178 22512 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1bdb4a0427c4400aaa3d458e9a0ec44e, State: Initialized, Role: FOLLOWER
08:04:37.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.611737 22512 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:37.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.612542 22519 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:37.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.612840 22519 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: This master's current role is: FOLLOWER
08:04:37.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.613481 22512 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: configured and running, proceeding with master startup.
08:04:37.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.623558 22530 catalog_manager.cc:1269] Loaded cluster ID: 40948aa4facf4848a044328dc5740242
08:04:37.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.624236 22530 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: loading cluster ID for follower catalog manager: success
08:04:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.629463 22530 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: acquiring CA information for follower catalog manager: success
08:04:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.632632 22530 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:37.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.704082 21428 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV73fp_/build/asan/bin/kudu with pid 21439
08:04:37.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.706863 22477 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 2 FOLLOWER]: Advancing to term 3
08:04:37.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.716486 22536 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "95c6cbdc462b459d91cf07e56273c510" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.717061 22536 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: This master's current role is: FOLLOWER
08:04:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.717478 22519 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 95c6cbdc462b459d91cf07e56273c510. Latest consensus state: current_term: 3 leader_uuid: "95c6cbdc462b459d91cf07e56273c510" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.717701 22519 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: This master's current role is: FOLLOWER
08:04:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.727427 22452 connection.cc:537] server connection from 127.0.0.1:38844 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:04:37.728 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.237.62:46031(127.20.237.62:46031)] handling channelInactive
08:04:37.728 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.237.62:46031(127.20.237.62:46031)] cleaning up while in state READY due to: connection closed
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.729034 22532 negotiation.cc:337] Failed RPC negotiation. Trace:
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:37.620515 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.20.237.62:46031 (local address 127.0.0.1:50232)
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:37.631672 (+ 11157us) negotiation.cc:107] Waiting for socket to connect
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:37.631716 (+ 44us) client_negotiation.cc:174] Beginning negotiation
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:37.631883 (+ 167us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:37.703458 (+ 71575us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:37.703469 (+ 11us) client_negotiation.cc:394] Received NEGOTIATE response from server
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:37.703515 (+ 46us) client_negotiation.cc:189] Negotiated authn=CERTIFICATE
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:37.704240 (+ 725us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:37.704255 (+ 15us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:37.727640 (+ 23385us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:04:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":10900,"thread_start_us":10750,"threads_started":1}
08:04:37.730 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:37.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.729962 22434 master_runner.cc:186] Error getting master registration for 127.20.237.62:46031: Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
08:04:37.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.733891 22434 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:04:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.995667 22537 raft_consensus.cc:493] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 95c6cbdc462b459d91cf07e56273c510)
08:04:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.995885 22537 raft_consensus.cc:515] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:37.996676 22537 leader_election.cc:290] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031), 1bdb4a0427c4400aaa3d458e9a0ec44e (127.20.237.61:45365)
08:04:37.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:37.999504 22304 leader_election.cc:336] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031): Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: connect: Connection refused (error 111)
08:04:38.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.004940 22477 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "36591742c42a4a36906a28f1b8029ece" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" is_pre_election: true
08:04:38.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.005846 22306 leader_election.cc:304] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 36591742c42a4a36906a28f1b8029ece; no voters: 1bdb4a0427c4400aaa3d458e9a0ec44e, 95c6cbdc462b459d91cf07e56273c510
08:04:38.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.006215 22537 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
08:04:38.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.013739 22519 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 95c6cbdc462b459d91cf07e56273c510)
08:04:38.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.014009 22519 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.015776 22519 leader_election.cc:290] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031), 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459)
08:04:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.016464 22452 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: connect: Connection refused (error 111)
08:04:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.018484 22452 leader_election.cc:336] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031): Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: connect: Connection refused (error 111)
08:04:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.023057 22329 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "36591742c42a4a36906a28f1b8029ece" is_pre_election: true
08:04:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.023272 22329 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1bdb4a0427c4400aaa3d458e9a0ec44e in term 3.
08:04:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.023831 22453 leader_election.cc:304] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1bdb4a0427c4400aaa3d458e9a0ec44e, 36591742c42a4a36906a28f1b8029ece; no voters: 95c6cbdc462b459d91cf07e56273c510
08:04:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.024181 22519 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 3 FOLLOWER]: Leader pre-election won for term 4
08:04:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.024302 22519 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 3 FOLLOWER]: Starting leader election (detected failure of leader 95c6cbdc462b459d91cf07e56273c510)
08:04:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.024405 22519 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 3 FOLLOWER]: Advancing to term 4
08:04:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.026432 22519 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:38.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.027012 22519 leader_election.cc:290] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 4 election: Requested vote from peers 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031), 36591742c42a4a36906a28f1b8029ece (127.20.237.60:44459)
08:04:38.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.027555 22329 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "36591742c42a4a36906a28f1b8029ece"
08:04:38.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.027724 22329 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 3 FOLLOWER]: Advancing to term 4
08:04:38.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.028229 22452 leader_election.cc:336] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031): Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: connect: Connection refused (error 111)
08:04:38.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.029850 22329 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1bdb4a0427c4400aaa3d458e9a0ec44e in term 4.
08:04:38.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.030309 22453 leader_election.cc:304] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1bdb4a0427c4400aaa3d458e9a0ec44e, 36591742c42a4a36906a28f1b8029ece; no voters: 95c6cbdc462b459d91cf07e56273c510
08:04:38.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.030661 22519 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 4 FOLLOWER]: Leader election won for term 4
08:04:38.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.030856 22519 raft_consensus.cc:697] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [term 4 LEADER]: Becoming Leader. State: Replica: 1bdb4a0427c4400aaa3d458e9a0ec44e, State: Running, Role: LEADER
08:04:38.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.031276 22519 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 3.13, Last appended by leader: 13, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } }
08:04:38.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.032522 22536 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1bdb4a0427c4400aaa3d458e9a0ec44e. Latest consensus state: current_term: 4 leader_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:38.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.032670 22536 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: This master's current role is: LEADER
08:04:38.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.033360 22544 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:04:38.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.035732 22544 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=5fd10891efd149429f1c33147a56eb32]
08:04:38.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.036253 22544 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19]
08:04:38.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.037829 22544 tablet_loader.cc:96] loaded metadata for tablet 08b057f94ae64be0a3d445c579c18414 (table txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19])
08:04:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.038189 22544 tablet_loader.cc:96] loaded metadata for tablet 0e1c9874fda544e8a1305b9a274ee9e2 (table txn_manager_fallback_rolling [id=6e2ee7d767ea40a7b365416d1fb98b19])
08:04:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.038487 22544 tablet_loader.cc:96] loaded metadata for tablet 39a9dffb524f44f6acbe35914817b65e (table kudu_system.kudu_transactions [id=5fd10891efd149429f1c33147a56eb32])
08:04:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.038695 22544 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:04:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.039523 22544 catalog_manager.cc:1269] Loaded cluster ID: 40948aa4facf4848a044328dc5740242
08:04:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.039616 22544 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:04:38.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.040715 22544 catalog_manager.cc:1514] Loading token signing keys...
08:04:38.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.041472 22544 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e: Loaded TSK: 0
08:04:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.042359 22544 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:04:38.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.147253 22452 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e -> Peer 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031): Couldn't send request to peer 95c6cbdc462b459d91cf07e56273c510. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:38.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.152612 22329 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [term 4 FOLLOWER]: Refusing update from remote peer 1bdb4a0427c4400aaa3d458e9a0ec44e: Log matching property violated. Preceding OpId in replica: term: 3 index: 13. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
08:04:38.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.153364 22536 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [LEADER]: Connected to new peer: Peer: permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
08:04:38.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.155114 22537 sys_catalog.cc:455] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1bdb4a0427c4400aaa3d458e9a0ec44e. Latest consensus state: current_term: 4 leader_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:38.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.155390 22537 sys_catalog.cc:458] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: This master's current role is: FOLLOWER
08:04:38.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.156520 22536 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:38.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.156759 22536 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e [sys.catalog]: This master's current role is: LEADER
08:04:38.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.157346 22537 sys_catalog.cc:455] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95c6cbdc462b459d91cf07e56273c510" member_type: VOTER last_known_addr { host: "127.20.237.62" port: 46031 } } peers { permanent_uuid: "1bdb4a0427c4400aaa3d458e9a0ec44e" member_type: VOTER last_known_addr { host: "127.20.237.61" port: 45365 } } peers { permanent_uuid: "36591742c42a4a36906a28f1b8029ece" member_type: VOTER last_known_addr { host: "127.20.237.60" port: 44459 } } }
08:04:38.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.157513 22537 sys_catalog.cc:458] T 00000000000000000000000000000000 P 36591742c42a4a36906a28f1b8029ece [sys.catalog]: This master's current role is: FOLLOWER
08:04:38.203 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.62:46031(127.20.237.62:46031)] connecting to peer
08:04:38.204 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031
08:04:38.204 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.237.62:46031(127.20.237.62:46031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031
08:04:38.204 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.20.237.62:46031(127.20.237.62:46031) for tablet Kudu Master: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031
08:04:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.216346 21879 heartbeater.cc:344] Connected to a master server at 127.20.237.61:45365
08:04:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.217983 22467 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a1b897c93d74495e9cb6f845c7fb09a8" instance_seqno: 1761206670258861) as {username='slave'} at 127.20.237.1:56885; Asking this server to re-register.
08:04:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.218602 21879 heartbeater.cc:461] Registering TS with master...
08:04:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.218885 21879 heartbeater.cc:507] Master 127.20.237.61:45365 requested a full tablet report, sending...
08:04:38.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.220551 22467 ts_manager.cc:194] Registered new tserver with Master: a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165)
08:04:38.223 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.62:46031(127.20.237.62:46031)] connecting to peer
08:04:38.223 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.20.237.61:45365(127.20.237.61:45365)] connecting to peer
08:04:38.224 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031
08:04:38.224 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.237.62:46031(127.20.237.62:46031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031
08:04:38.224 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.237.62:46031: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031
08:04:38.224 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.237.61:45365(127.20.237.61:45365)] Successfully connected to peer
08:04:38.235 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7e1ac601, L:/127.0.0.1:38876 - R:/127.20.237.61:45365] using TOKEN/null
08:04:38.237 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:04:38.237 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=34), Traces: [0ms] sending RPC to server master-127.20.237.62:46031, [1ms] received response from server master-127.20.237.62:46031: Network error: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [21ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: Network error: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031, [24ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [33ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: OK, deferred=Deferred@2027719563(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@61e865fe, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@6c3d33a8)) after lookup
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.269686 22467 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38886:
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:04:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:04:38.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.272814 22549 master.cc:464] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 1 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
08:04:38.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.450416 22021 heartbeater.cc:344] Connected to a master server at 127.20.237.61:45365
08:04:38.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.451591 22467 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "90e5f9f2bb4d4a80b42c34aba660acbb" instance_seqno: 1761206670796356) as {username='slave'} at 127.20.237.2:37121; Asking this server to re-register.
08:04:38.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.452092 22021 heartbeater.cc:461] Registering TS with master...
08:04:38.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.452312 22021 heartbeater.cc:507] Master 127.20.237.61:45365 requested a full tablet report, sending...
08:04:38.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.453361 22467 ts_manager.cc:194] Registered new tserver with Master: 90e5f9f2bb4d4a80b42c34aba660acbb (127.20.237.2:43461)
08:04:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.454223 22497 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:38876 (request call id 1) took 216 ms (client timeout 216 ms). Trace:
08:04:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.454368 22497 rpcz_store.cc:269] 1023 08:04:38.237919 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:04:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:38.237994 (+ 75us) service_pool.cc:225] Handling call
08:04:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:38.454165 (+216171us) inbound_call.cc:173] Queueing success response
08:04:38.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":612,"init.queue_time_us":356,"thread_start_us":981,"threads_started":5}
08:04:38.455 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=3, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.20.237.62:46031, [1ms] received response from server master-127.20.237.62:46031: Network error: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [21ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: Network error: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031, [24ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [33ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: OK, [34ms] sending RPC to server master-127.20.237.61:45365, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@2027719563(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@61e865fe -> (continuation of Deferred@1259182175 after retry RPC@1665971968), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@6c3d33a8 -> (continuation of Deferred@1259182175 after retry RPC@1665971968)))
08:04:38.456 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=3, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.20.237.62:46031, [1ms] received response from server master-127.20.237.62:46031: Network error: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.62:46031, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.61:45365, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.237.60:44459, [21ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.62:46031: Network error: Failed to connect to peer master-127.20.237.62:46031(127.20.237.62:46031): Connection refused: /127.20.237.62:46031, [24ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.60:44459: OK, [33ms] Sub RPC ConnectToMaster: received response from server master-127.20.237.61:45365: OK, [34ms] sending RPC to server master-127.20.237.61:45365, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@2027719563(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@61e865fe -> (continuation of Deferred@1259182175 after retry RPC@1665971968), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@6c3d33a8 -> (continuation of Deferred@1259182175 after retry RPC@1665971968)))
08:04:38.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.608063 22467 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "59f9188ecd9f41c5978c24cb0e7b8d0c" instance_seqno: 1761206671308578) as {username='slave'} at 127.20.237.3:40089; Asking this server to re-register.
08:04:38.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.608798 22160 heartbeater.cc:461] Registering TS with master...
08:04:38.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.609050 22160 heartbeater.cc:507] Master 127.20.237.61:45365 requested a full tablet report, sending...
08:04:38.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.610357 22467 ts_manager.cc:194] Registered new tserver with Master: 59f9188ecd9f41c5978c24cb0e7b8d0c (127.20.237.3:37041)
08:04:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.652266 22452 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e -> Peer 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031): Couldn't send request to peer 95c6cbdc462b459d91cf07e56273c510. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:04:38.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.713619 22497 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:38876 (request call id 2) took 250 ms (client timeout 250 ms). Trace:
08:04:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.713742 22497 rpcz_store.cc:269] 1023 08:04:38.463416 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:04:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:38.463483 (+ 67us) service_pool.cc:225] Handling call
08:04:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:38.713552 (+250069us) inbound_call.cc:173] Queueing success response
08:04:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
08:04:38.714 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.20.237.61:45365, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1137253547(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@48443865, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@6f2c8e52))
08:04:38.714 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.20.237.61:45365, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1137253547(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@48443865, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@6f2c8e52))
08:04:38.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:38.760468 22043 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.058s from last keepalive heartbeat (effective timeout is 1.000s)
08:04:38.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.761729 22066 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.20.237.62:46031)
08:04:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.772342 22559 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
08:04:38.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.779237 22563 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
08:04:38.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.973848 22497 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:38876 (request call id 3) took 250 ms (client timeout 250 ms). Trace:
08:04:38.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:38.974005 22497 rpcz_store.cc:269] 1023 08:04:38.723665 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:04:38.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:38.723738 (+ 73us) service_pool.cc:225] Handling call
08:04:38.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:38.973770 (+250032us) inbound_call.cc:173] Queueing success response
08:04:38.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
08:04:38.975 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.20.237.61:45365, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1652143556(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@b173efe, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@427fd0))
08:04:38.975 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.20.237.61:45365, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1652143556(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@b173efe, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@427fd0))
08:04:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:39.146016 22452 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e -> Peer 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031): Couldn't send request to peer 95c6cbdc462b459d91cf07e56273c510. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
08:04:39.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:39.233495 22497 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:38876 (request call id 4) took 250 ms (client timeout 250 ms). Trace:
08:04:39.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:39.233608 22497 rpcz_store.cc:269] 1023 08:04:38.983344 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:04:39.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:38.983415 (+ 71us) service_pool.cc:225] Handling call
08:04:39.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:04:39.233431 (+250016us) inbound_call.cc:173] Queueing success response
08:04:39.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
08:04:39.234 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.20.237.61:45365, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@520347928(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@30fba0a9, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@2a98886))
08:04:39.234 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.20.237.61:45365, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@520347928(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1771505555@30fba0a9, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/263344142@2a98886))
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:39.308053 22466 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:47334:
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:04:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:39.308928 22466 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:04:39.324 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 59f9188ecd9f41c5978c24cb0e7b8d0c (127.20.237.3:37041): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1761206672 last_transition_timestamp: 1761206678
08:04:39.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:39.588670 22452 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e -> Peer 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031): Couldn't send request to peer 95c6cbdc462b459d91cf07e56273c510. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
08:04:39.734 [DEBUG - main] (Connection.java:575) [peer 59f9188ecd9f41c5978c24cb0e7b8d0c(127.20.237.3:37041)] disconnecting while in state READY
08:04:39.734 [DEBUG - main] (Connection.java:575) [peer 90e5f9f2bb4d4a80b42c34aba660acbb(127.20.237.2:43461)] disconnecting while in state READY
08:04:39.734 [DEBUG - main] (Connection.java:575) [peer master-127.20.237.60:44459(127.20.237.60:44459)] disconnecting while in state READY
08:04:39.734 [DEBUG - main] (Connection.java:575) [peer master-127.20.237.61:45365(127.20.237.61:45365)] disconnecting while in state READY
08:04:39.734 [DEBUG - main] (Connection.java:575) [peer master-127.20.237.62:46031(127.20.237.62:46031)] disconnecting while in state TERMINATED
08:04:39.735 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 59f9188ecd9f41c5978c24cb0e7b8d0c(127.20.237.3:37041)] handling channelInactive
08:04:39.735 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 59f9188ecd9f41c5978c24cb0e7b8d0c(127.20.237.3:37041)] cleaning up while in state READY due to: connection closed
08:04:39.736 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:04:39.736 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 90e5f9f2bb4d4a80b42c34aba660acbb(127.20.237.2:43461)] handling channelInactive
08:04:39.736 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 90e5f9f2bb4d4a80b42c34aba660acbb(127.20.237.2:43461)] cleaning up while in state READY due to: connection closed
08:04:39.736 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.237.60:44459(127.20.237.60:44459)] handling channelInactive
08:04:39.736 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.237.60:44459(127.20.237.60:44459)] cleaning up while in state READY due to: connection closed
08:04:39.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:39.736867 21428 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV73fp_/build/asan/bin/kudu pid 21719
08:04:39.737 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.237.61:45365(127.20.237.61:45365)] handling channelInactive
08:04:39.737 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.237.61:45365(127.20.237.61:45365)] cleaning up while in state READY due to: connection closed
08:04:39.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:39.744818 21872 generic_service.cc:196] Checking for leaks (request via RPC)
08:04:39.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:39.843746 21428 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV73fp_/build/asan/bin/kudu with pid 21719
08:04:39.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:39.866925 21428 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV73fp_/build/asan/bin/kudu pid 21881
08:04:39.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:39.873924 22013 generic_service.cc:196] Checking for leaks (request via RPC)
08:04:39.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:39.912895 22038 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.237.1:34165: connect: Connection refused (error 111)
08:04:39.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:39.914212 22038 consensus_peers.cc:597] T 08b057f94ae64be0a3d445c579c18414 P 59f9188ecd9f41c5978c24cb0e7b8d0c -> Peer a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165): Couldn't send request to peer a1b897c93d74495e9cb6f845c7fb09a8. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.1:34165: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:39.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:39.966889 22038 consensus_peers.cc:597] T 39a9dffb524f44f6acbe35914817b65e P 59f9188ecd9f41c5978c24cb0e7b8d0c -> Peer a1b897c93d74495e9cb6f845c7fb09a8 (127.20.237.1:34165): Couldn't send request to peer a1b897c93d74495e9cb6f845c7fb09a8. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.1:34165: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:39.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:39.973922 21428 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV73fp_/build/asan/bin/kudu with pid 21881
08:04:39.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:39.991849 21428 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV73fp_/build/asan/bin/kudu pid 22020
08:04:39.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:39.998230 22153 generic_service.cc:196] Checking for leaks (request via RPC)
08:04:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:40.050614 22285 master_runner.cc:186] Error getting master registration for 127.20.237.62:46031: Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: connect: Connection refused (error 111),
08:04:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:40.050781 22285 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:04:40.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:40.102777 22452 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1bdb4a0427c4400aaa3d458e9a0ec44e -> Peer 95c6cbdc462b459d91cf07e56273c510 (127.20.237.62:46031): Couldn't send request to peer 95c6cbdc462b459d91cf07e56273c510. Status: Network error: Client connection negotiation failed: client connection to 127.20.237.62:46031: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
08:04:40.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:40.123308 21428 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV73fp_/build/asan/bin/kudu with pid 22020
08:04:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:40.144474 21428 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV73fp_/build/asan/bin/kudu pid 22434
08:04:40.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:40.150103 22507 generic_service.cc:196] Checking for leaks (request via RPC)
08:04:40.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:40.263103 21428 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV73fp_/build/asan/bin/kudu with pid 22434
08:04:40.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:04:40.284507 22306 connection.cc:537] server connection from 127.0.0.1:57726 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:04:40.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:40.284819 21428 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskV73fp_/build/asan/bin/kudu pid 22285
08:04:40.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:40.290671 22359 generic_service.cc:196] Checking for leaks (request via RPC)
08:04:40.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:04:40.396888 21428 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskV73fp_/build/asan/bin/kudu with pid 22285
08:04:40.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T08:04:40Z chronyd exiting
08:04:40.551 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)