09:04:52.180 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
09:04:52.180 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskWDBSYb/test-tmp
09:04:52.181 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
09:04:52.181 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskWDBSYb/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
09:04:52.182 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
09:04:52.493 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:04:52.493 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:04:52.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.495682 28331 mini_oidc.cc:150] Starting JWKS server
09:04:52.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.497958 28331 webserver.cc:492] Webserver started at http://127.0.0.1:33495/ using document root <none> and password file <none>
09:04:52.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.498185 28331 mini_oidc.cc:186] Starting OIDC Discovery server
09:04:52.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.498669 28331 webserver.cc:492] Webserver started at http://127.0.0.1:42719/ using document root <none> and password file <none>
09:04:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-12T09:04:52Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:04:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-12T09:04:52Z Disabled control of system clock
09:04:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.525671 28331 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal
09:04:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data
09:04:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/logs
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data/info.pb
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.254:35867
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.254
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33495/jwks.json
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:04:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:04:52.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.832160 28342 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:52.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.832474 28342 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:52.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.832559 28342 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.840472 28342 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:04:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.840555 28342 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:04:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.840600 28342 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.840637 28342 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:04:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.840694 28342 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:04:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.840732 28342 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:04:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.840780 28342 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33495/jwks.json
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.850205 28342 master_runner.cc:386] Master server non-default flags:
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.254:35867
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33495/jwks.json
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data/info.pb
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.254
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/logs
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:52.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:04:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:04:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 08:43:22 UTC on 24a791456cd2
09:04:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9476
09:04:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.851665 28342 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:52.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.853360 28342 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:52.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.860458 28347 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:52.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.860994 28348 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:52.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.861250 28350 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:52.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.861737 28342 server_base.cc:1047] running on GCE node
09:04:52.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.862387 28342 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:52.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.863600 28342 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:52.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.864794 28342 hybrid_clock.cc:648] HybridClock initialized: now 1765530292864731 us; error 46 us; skew 500 ppm
09:04:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.865046 28342 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.866997 28342 webserver.cc:492] Webserver started at http://127.27.170.254:46879/ using document root <none> and password file <none>
09:04:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.867599 28342 fs_manager.cc:362] Metadata directory not provided
09:04:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.867697 28342 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.867969 28342 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:52.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.870117 28342 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data/instance:
09:04:52.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e57ee419ab3c4ad2811a115333f3909f"
09:04:52.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:52 on dist-test-slave-hgx1"
09:04:52.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.870779 28342 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal/instance:
09:04:52.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e57ee419ab3c4ad2811a115333f3909f"
09:04:52.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:52 on dist-test-slave-hgx1"
09:04:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.875384 28342 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.004s sys 0.002s
09:04:52.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.878320 28356 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.879458 28342 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
09:04:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.879580 28342 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data,/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal
09:04:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e57ee419ab3c4ad2811a115333f3909f"
09:04:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:52 on dist-test-slave-hgx1"
09:04:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.879750 28342 fs_report.cc:389] FS layout report
09:04:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal
09:04:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal
09:04:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data/data
09:04:52.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:52.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.899094 28342 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:52.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.899870 28342 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:52.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.900167 28342 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:52.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.921257 28342 rpc_server.cc:307] RPC server started. Bound to: 127.27.170.254:35867
09:04:52.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.921290 28417 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.170.254:35867 every 8 connection(s)
09:04:52.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.922716 28342 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data/info.pb
09:04:52.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.926637 28418 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:52.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.931205 28331 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu as pid 28342
09:04:52.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.931491 28331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal/instance
09:04:52.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.932001 28418 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } has no permanent_uuid. Determining permanent_uuid...
09:04:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.933671 28331 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/wal
09:04:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/data
09:04:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/logs
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/data/info.pb
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.253:39067
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.253
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33495/jwks.json
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:04:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:04:52.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:52.948127 28418 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } has no permanent_uuid. Determining permanent_uuid...
09:04:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.949235 28357 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111)
09:04:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:52.950747 28418 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.253:39067: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111)
09:04:53.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.000094 28418 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } attempt: 1
09:04:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.001955 28418 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.253:39067: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111)
09:04:53.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.070191 28418 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } attempt: 2
09:04:53.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.072064 28418 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.253:39067: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111)
09:04:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.163305 28418 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } attempt: 3
09:04:53.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.165102 28418 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.253:39067: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111)
09:04:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.239818 28422 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.240154 28422 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.240267 28422 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.248230 28422 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:04:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.248332 28422 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:04:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.248406 28422 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.248468 28422 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:04:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.248536 28422 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:04:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.248592 28422 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:04:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.248653 28422 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33495/jwks.json
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.258217 28422 master_runner.cc:386] Master server non-default flags:
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/data
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/wal
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.253:39067
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33495/jwks.json
09:04:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/data/info.pb
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.253
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/logs
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 08:43:22 UTC on 24a791456cd2
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9476
09:04:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.260172 28422 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:53.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.261940 28422 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.269184 28429 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.269214 28428 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.269948 28431 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.270123 28422 server_base.cc:1047] running on GCE node
09:04:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.270818 28422 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:53.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.272101 28422 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:53.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.273300 28422 hybrid_clock.cc:648] HybridClock initialized: now 1765530293273233 us; error 50 us; skew 500 ppm
09:04:53.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.273576 28422 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:53.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.275643 28422 webserver.cc:492] Webserver started at http://127.27.170.253:34387/ using document root <none> and password file <none>
09:04:53.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.276329 28422 fs_manager.cc:362] Metadata directory not provided
09:04:53.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.276438 28422 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:53.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.276705 28422 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.278869 28422 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/data/instance:
09:04:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf0667fb0e154a5f82650259c7e41f1a"
09:04:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:53 on dist-test-slave-hgx1"
09:04:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.279542 28422 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/wal/instance:
09:04:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf0667fb0e154a5f82650259c7e41f1a"
09:04:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:53 on dist-test-slave-hgx1"
09:04:53.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.284518 28422 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.003s sys 0.004s
09:04:53.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.287580 28437 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.288883 28422 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.289013 28422 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/data,/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/wal
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf0667fb0e154a5f82650259c7e41f1a"
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:53 on dist-test-slave-hgx1"
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.289155 28422 fs_report.cc:389] FS layout report
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/wal
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/wal
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/data/data
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.305009 28422 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.305795 28422 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.306030 28422 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:53.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.308329 28418 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } attempt: 4
09:04:53.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.310232 28418 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.253:39067: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111)
09:04:53.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.327432 28422 rpc_server.cc:307] RPC server started. Bound to: 127.27.170.253:39067
09:04:53.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.327508 28498 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.170.253:39067 every 8 connection(s)
09:04:53.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.329001 28422 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/data/info.pb
09:04:53.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.330145 28331 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu as pid 28422
09:04:53.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.330304 28331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-1/wal/instance
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.331494 28331 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/logs
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data/info.pb
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.252:40751
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.252
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33495/jwks.json
09:04:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:04:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:04:53.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.334378 28499 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:53.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.340610 28499 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } has no permanent_uuid. Determining permanent_uuid...
09:04:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.353289 28499 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } has no permanent_uuid. Determining permanent_uuid...
09:04:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.361621 28499 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } has no permanent_uuid. Determining permanent_uuid...
09:04:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.362591 28438 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.364066 28499 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.252:40751: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.413333 28499 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } attempt: 1
09:04:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.415231 28499 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.252:40751: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:53.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.483426 28499 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } attempt: 2
09:04:53.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.485512 28499 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.252:40751: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:53.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.576749 28499 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } attempt: 3
09:04:53.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.578842 28499 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.252:40751: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:53.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.609609 28418 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } attempt: 5
09:04:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.616027 28418 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } has no permanent_uuid. Determining permanent_uuid...
09:04:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.617753 28418 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.252:40751: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:53.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.648505 28501 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:53.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.648799 28501 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:53.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.648878 28501 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.656893 28501 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:04:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.656983 28501 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:04:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.657027 28501 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.657065 28501 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:04:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.657114 28501 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:04:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.657150 28501 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:04:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.657191 28501 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33495/jwks.json
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.667109 28501 master_runner.cc:386] Master server non-default flags:
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.252:40751
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33495/jwks.json
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data/info.pb
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.252
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/logs
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 08:43:22 UTC on 24a791456cd2
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9476
09:04:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.668838 28501 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.668991 28418 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } attempt: 1
09:04:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.670779 28501 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.670790 28418 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.252:40751: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.678376 28509 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.678406 28510 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.679534 28512 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.679896 28501 server_base.cc:1047] running on GCE node
09:04:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.680622 28501 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:53.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.681982 28501 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:53.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.683204 28501 hybrid_clock.cc:648] HybridClock initialized: now 1765530293683138 us; error 40 us; skew 500 ppm
09:04:53.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.683550 28501 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:53.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.685693 28501 webserver.cc:492] Webserver started at http://127.27.170.252:35295/ using document root <none> and password file <none>
09:04:53.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.686362 28501 fs_manager.cc:362] Metadata directory not provided
09:04:53.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.686486 28501 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:53.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.686831 28501 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:53.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.689388 28501 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data/instance:
09:04:53.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b405358c0e04b69ba905ce322ad556e"
09:04:53.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:53 on dist-test-slave-hgx1"
09:04:53.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.690141 28501 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal/instance:
09:04:53.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b405358c0e04b69ba905ce322ad556e"
09:04:53.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:53 on dist-test-slave-hgx1"
09:04:53.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.695259 28501 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
09:04:53.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.698428 28518 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:53.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.699602 28501 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
09:04:53.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.699760 28501 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data,/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b405358c0e04b69ba905ce322ad556e"
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:53 on dist-test-slave-hgx1"
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.699934 28501 fs_report.cc:389] FS layout report
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data/data
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:53.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.722071 28499 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } attempt: 4
09:04:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.723775 28499 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.252:40751: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.723945 28501 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.724815 28501 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:53.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.725085 28501 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:53.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.738993 28418 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } attempt: 2
09:04:53.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.741039 28418 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.170.252:40751: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:53.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.747047 28501 rpc_server.cc:307] RPC server started. Bound to: 127.27.170.252:40751
09:04:53.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.747079 28579 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.170.252:40751 every 8 connection(s)
09:04:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.748595 28501 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data/info.pb
09:04:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.752593 28580 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.757730 28331 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu as pid 28501
09:04:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.757902 28331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal/instance
09:04:53.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.758093 28580 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } has no permanent_uuid. Determining permanent_uuid...
09:04:53.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.770790 28580 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } has no permanent_uuid. Determining permanent_uuid...
09:04:53.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.778440 28580 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } has no permanent_uuid. Determining permanent_uuid...
09:04:53.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.790606 28580 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: Bootstrap starting.
09:04:53.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.794039 28580 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: Neither blocks nor log segments found. Creating new log.
09:04:53.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.795082 28580 log.cc:826] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: Log is configured to *not* fsync() on all Append() calls
09:04:53.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.797740 28580 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: No bootstrap required, opened a new log
09:04:53.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.802552 28580 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:53.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.803210 28580 raft_consensus.cc:385] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:53.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.803339 28580 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b405358c0e04b69ba905ce322ad556e, State: Initialized, Role: FOLLOWER
09:04:53.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.803979 28580 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:53.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.804963 28589 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:53.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.805238 28589 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: This master's current role is: FOLLOWER
09:04:53.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.806006 28580 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: configured and running, proceeding with master startup.
09:04:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.814024 28600 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:04:53.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.814128 28600 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:04:53.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.812218 28501 master_runner.cc:186] Error getting master registration for 127.27.170.254:35867: OK, instance_id { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" instance_seqno: 1765530292916264 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:04:53.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.818451 28501 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:04:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.839598 28589 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.839885 28589 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.841809 28589 leader_election.cc:290] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867), bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067)
09:04:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.847352 28418 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } attempt: 3
09:04:53.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.859436 28418 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: Bootstrap starting.
09:04:53.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.863693 28418 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: Neither blocks nor log segments found. Creating new log.
09:04:53.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.865099 28418 log.cc:826] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: Log is configured to *not* fsync() on all Append() calls
09:04:53.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.868182 28418 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: No bootstrap required, opened a new log
09:04:53.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.875490 28418 raft_consensus.cc:359] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:53.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.876003 28418 raft_consensus.cc:385] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:53.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.876160 28418 raft_consensus.cc:740] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e57ee419ab3c4ad2811a115333f3909f, State: Initialized, Role: FOLLOWER
09:04:53.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.876869 28418 consensus_queue.cc:260] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.877846 28603 sys_catalog.cc:455] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.878249 28603 sys_catalog.cc:458] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: This master's current role is: FOLLOWER
09:04:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.879168 28418 sys_catalog.cc:565] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: configured and running, proceeding with master startup.
09:04:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.879215 28383 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b405358c0e04b69ba905ce322ad556e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e57ee419ab3c4ad2811a115333f3909f" is_pre_election: true
09:04:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.879686 28383 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8b405358c0e04b69ba905ce322ad556e in term 0.
09:04:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.880301 28522 leader_election.cc:304] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8b405358c0e04b69ba905ce322ad556e, e57ee419ab3c4ad2811a115333f3909f; no voters:
09:04:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.880748 28589 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 0 FOLLOWER]: Leader pre-election won for term 1
09:04:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.880904 28589 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:04:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.881018 28589 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 0 FOLLOWER]: Advancing to term 1
09:04:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.883103 28589 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.884235 28589 leader_election.cc:290] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [CANDIDATE]: Term 1 election: Requested vote from peers e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867), bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067)
09:04:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.884380 28383 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b405358c0e04b69ba905ce322ad556e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e57ee419ab3c4ad2811a115333f3909f"
09:04:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.884666 28383 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 0 FOLLOWER]: Advancing to term 1
09:04:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.887662 28383 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8b405358c0e04b69ba905ce322ad556e in term 1.
09:04:53.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.888269 28522 leader_election.cc:304] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8b405358c0e04b69ba905ce322ad556e, e57ee419ab3c4ad2811a115333f3909f; no voters:
09:04:53.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.888713 28589 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 1 FOLLOWER]: Leader election won for term 1
09:04:53.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.900823 28589 raft_consensus.cc:697] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 1 LEADER]: Becoming Leader. State: Replica: 8b405358c0e04b69ba905ce322ad556e, State: Running, Role: LEADER
09:04:53.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.901470 28589 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:53.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.902565 28362 tablet.cc:2378] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:04:53.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.906903 28607 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b405358c0e04b69ba905ce322ad556e. Latest consensus state: current_term: 1 leader_uuid: "8b405358c0e04b69ba905ce322ad556e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:53.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.907413 28607 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: This master's current role is: LEADER
09:04:53.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.908582 28611 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:04:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.910676 28611 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:04:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.919389 28383 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 1 FOLLOWER]: Refusing update from remote peer 8b405358c0e04b69ba905ce322ad556e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:04:53.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.920400 28607 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [LEADER]: Connected to new peer: Peer: permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:53.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.928224 28342 master_runner.cc:186] Error getting master registration for 127.27.170.253:39067: OK, instance_id { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" instance_seqno: 1765530293322528 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:04:53.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.934945 28342 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:04:53.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.951143 28618 catalog_manager.cc:1568] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:04:53.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:53.951822 28618 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:04:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.958912 28603 sys_catalog.cc:455] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b405358c0e04b69ba905ce322ad556e. Latest consensus state: current_term: 1 leader_uuid: "8b405358c0e04b69ba905ce322ad556e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.959153 28603 sys_catalog.cc:458] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: This master's current role is: FOLLOWER
09:04:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.968786 28607 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8b405358c0e04b69ba905ce322ad556e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.969060 28607 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: This master's current role is: LEADER
09:04:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.973384 28611 catalog_manager.cc:1357] Generated new cluster ID: 36dfe5a18b044fa391749e4f6be4af0c
09:04:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.973501 28611 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:04:53.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.982333 28603 sys_catalog.cc:455] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8b405358c0e04b69ba905ce322ad556e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:53.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.982551 28603 sys_catalog.cc:458] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: This master's current role is: FOLLOWER
09:04:53.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:53.999257 28611 catalog_manager.cc:1380] Generated new certificate authority record
09:04:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.002583 28611 catalog_manager.cc:1514] Loading token signing keys...
09:04:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.023466 28499 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } attempt: 5
09:04:54.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.032086 28611 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: Generated new TSK 0
09:04:54.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.034765 28611 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:04:54.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.040938 28499 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a: Bootstrap starting.
09:04:54.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.045280 28499 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a: Neither blocks nor log segments found. Creating new log.
09:04:54.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.050076 28499 log.cc:826] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a: Log is configured to *not* fsync() on all Append() calls
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.050963 28331 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/wal
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/data
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/logs
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/data/info.pb
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.193:0
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.170.193
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.193
09:04:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:04:54.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.054121 28499 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a: No bootstrap required, opened a new log
09:04:54.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.061057 28499 raft_consensus.cc:359] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:54.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.061498 28499 raft_consensus.cc:385] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:54.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.061707 28499 raft_consensus.cc:740] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bf0667fb0e154a5f82650259c7e41f1a, State: Initialized, Role: FOLLOWER
09:04:54.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.062592 28499 consensus_queue.cc:260] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.069711 28626 sys_catalog.cc:455] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:54.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.071918 28626 sys_catalog.cc:458] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: This master's current role is: FOLLOWER
09:04:54.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.071789 28499 sys_catalog.cc:565] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: configured and running, proceeding with master startup.
09:04:54.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.072170 28463 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b405358c0e04b69ba905ce322ad556e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bf0667fb0e154a5f82650259c7e41f1a"
09:04:54.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.073052 28464 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b405358c0e04b69ba905ce322ad556e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bf0667fb0e154a5f82650259c7e41f1a" is_pre_election: true
09:04:54.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.073539 28462 raft_consensus.cc:3060] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 0 FOLLOWER]: Advancing to term 1
09:04:54.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.076908 28462 raft_consensus.cc:1275] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 1 FOLLOWER]: Refusing update from remote peer 8b405358c0e04b69ba905ce322ad556e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:04:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.077729 28607 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [LEADER]: Connected to new peer: Peer: permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.125484 28631 sys_catalog.cc:455] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8b405358c0e04b69ba905ce322ad556e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.125826 28631 sys_catalog.cc:458] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: This master's current role is: FOLLOWER
09:04:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.126276 28626 sys_catalog.cc:455] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b405358c0e04b69ba905ce322ad556e. Latest consensus state: current_term: 1 leader_uuid: "8b405358c0e04b69ba905ce322ad556e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.126485 28626 sys_catalog.cc:458] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: This master's current role is: FOLLOWER
09:04:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.161906 28607 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8b405358c0e04b69ba905ce322ad556e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.162155 28607 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: This master's current role is: LEADER
09:04:54.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.175765 28642 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:04:54.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.468499 28625 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:54.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.468928 28625 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:54.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.469072 28625 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.481992 28625 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.482177 28625 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:04:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.482239 28625 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:04:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.482282 28625 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:04:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.482391 28625 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.170.193
09:04:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.497529 28625 tablet_server_runner.cc:78] Tablet server non-default flags:
09:04:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/data
09:04:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/wal
09:04:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.193:0
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/data/info.pb
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.193
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.170.193
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/logs
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 08:43:22 UTC on 24a791456cd2
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9476
09:04:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.499737 28625 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.502135 28625 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:54.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.514091 28647 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.515817 28648 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.516714 28650 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.516674 28625 server_base.cc:1047] running on GCE node
09:04:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.517346 28625 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:54.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.518218 28625 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.519444 28625 hybrid_clock.cc:648] HybridClock initialized: now 1765530294519378 us; error 70 us; skew 500 ppm
09:04:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.519752 28625 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.525655 28625 webserver.cc:492] Webserver started at http://127.27.170.193:38651/ using document root <none> and password file <none>
09:04:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.526453 28625 fs_manager.cc:362] Metadata directory not provided
09:04:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.526574 28625 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.526907 28625 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.529289 28625 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/data/instance:
09:04:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2cad8cbd220442daa5b0ca7df42e0875"
09:04:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:54 on dist-test-slave-hgx1"
09:04:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.530046 28625 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/wal/instance:
09:04:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2cad8cbd220442daa5b0ca7df42e0875"
09:04:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:54 on dist-test-slave-hgx1"
09:04:54.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.535203 28625 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
09:04:54.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.538821 28656 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.540355 28625 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.540524 28625 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/data,/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/wal
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2cad8cbd220442daa5b0ca7df42e0875"
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:54 on dist-test-slave-hgx1"
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.540699 28625 fs_report.cc:389] FS layout report
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/wal
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/wal
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/data/data
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:54.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:54.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:54.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:54.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:54.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:54.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.577519 28625 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:54.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.578339 28625 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:54.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.578590 28625 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:54.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.581571 28625 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:04:54.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.581703 28625 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:04:54.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.581817 28625 ts_tablet_manager.cc:616] Registered 0 tablets
09:04:54.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.581899 28625 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:04:54.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.643996 28625 rpc_server.cc:307] RPC server started. Bound to: 127.27.170.193:38707
09:04:54.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.644075 28780 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.170.193:38707 every 8 connection(s)
09:04:54.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.645282 28625 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/data/info.pb
09:04:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.647547 28782 heartbeater.cc:344] Connected to a master server at 127.27.170.254:35867
09:04:54.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.648026 28782 heartbeater.cc:461] Registering TS with master...
09:04:54.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.650067 28782 heartbeater.cc:507] Master 127.27.170.254:35867 requested a full tablet report, sending...
09:04:54.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.652468 28331 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu as pid 28625
09:04:54.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.652660 28331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-0/wal/instance
09:04:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.658066 28372 ts_manager.cc:194] Registered new tserver with Master: 2cad8cbd220442daa5b0ca7df42e0875 (127.27.170.193:38707)
09:04:54.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.659907 28781 heartbeater.cc:344] Connected to a master server at 127.27.170.252:40751
09:04:54.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.660069 28781 heartbeater.cc:461] Registering TS with master...
09:04:54.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.660494 28781 heartbeater.cc:507] Master 127.27.170.252:40751 requested a full tablet report, sending...
09:04:54.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.660750 28783 heartbeater.cc:344] Connected to a master server at 127.27.170.253:39067
09:04:54.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.660867 28783 heartbeater.cc:461] Registering TS with master...
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.660904 28331 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/wal
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/data
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/logs
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/data/info.pb
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.194:0
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.170.194
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.194
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:04:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.661303 28783 heartbeater.cc:507] Master 127.27.170.253:39067 requested a full tablet report, sending...
09:04:54.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.662065 28534 ts_manager.cc:194] Registered new tserver with Master: 2cad8cbd220442daa5b0ca7df42e0875 (127.27.170.193:38707)
09:04:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.662838 28454 ts_manager.cc:194] Registered new tserver with Master: 2cad8cbd220442daa5b0ca7df42e0875 (127.27.170.193:38707)
09:04:54.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.664011 28534 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.170.193:45607
09:04:54.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.953902 28618 catalog_manager.cc:1269] Loaded cluster ID: 36dfe5a18b044fa391749e4f6be4af0c
09:04:54.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.953996 28618 catalog_manager.cc:1562] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: loading cluster ID for follower catalog manager: success
09:04:54.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.956593 28618 catalog_manager.cc:1584] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: acquiring CA information for follower catalog manager: success
09:04:54.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.958240 28618 catalog_manager.cc:1612] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:04:54.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.977105 28786 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:54.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.977351 28786 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:54.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.977452 28786 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.985371 28786 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.985491 28786 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:04:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.985541 28786 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:04:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.985591 28786 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:04:54.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:54.985693 28786 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.170.194
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.994912 28786 tablet_server_runner.cc:78] Tablet server non-default flags:
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/data
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/wal
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.194:0
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/data/info.pb
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.194
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.170.194
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/logs
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 08:43:22 UTC on 24a791456cd2
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9476
09:04:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:54.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.996420 28786 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:54.998041 28786 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.006520 28792 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.006526 28791 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:55.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.007524 28794 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:55.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.007906 28786 server_base.cc:1047] running on GCE node
09:04:55.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.008452 28786 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:55.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.009281 28786 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:55.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.010500 28786 hybrid_clock.cc:648] HybridClock initialized: now 1765530295010446 us; error 35 us; skew 500 ppm
09:04:55.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.010751 28786 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:55.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.012935 28786 webserver.cc:492] Webserver started at http://127.27.170.194:34817/ using document root <none> and password file <none>
09:04:55.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.013643 28786 fs_manager.cc:362] Metadata directory not provided
09:04:55.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.013772 28786 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:55.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.014096 28786 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:55.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.016387 28786 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/data/instance:
09:04:55.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a6c1e50223f43bdbd861350b1ab79d6"
09:04:55.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:55 on dist-test-slave-hgx1"
09:04:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.017107 28786 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/wal/instance:
09:04:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a6c1e50223f43bdbd861350b1ab79d6"
09:04:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:55 on dist-test-slave-hgx1"
09:04:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.022306 28786 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
09:04:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.025513 28800 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:55.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.026708 28786 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.026849 28786 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/data,/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/wal
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a6c1e50223f43bdbd861350b1ab79d6"
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:55 on dist-test-slave-hgx1"
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.027045 28786 fs_report.cc:389] FS layout report
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/wal
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/wal
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/data/data
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:55.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.049273 28786 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.050071 28786 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.050338 28786 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:55.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.053202 28786 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:04:55.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.053309 28786 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:04:55.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.053452 28786 ts_tablet_manager.cc:616] Registered 0 tablets
09:04:55.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.053536 28786 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:04:55.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.107654 28786 rpc_server.cc:307] RPC server started. Bound to: 127.27.170.194:43093
09:04:55.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.107703 28920 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.170.194:43093 every 8 connection(s)
09:04:55.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.108960 28786 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/data/info.pb
09:04:55.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.112116 28922 heartbeater.cc:344] Connected to a master server at 127.27.170.254:35867
09:04:55.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.112496 28922 heartbeater.cc:461] Registering TS with master...
09:04:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.113394 28922 heartbeater.cc:507] Master 127.27.170.254:35867 requested a full tablet report, sending...
09:04:55.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.115010 28372 ts_manager.cc:194] Registered new tserver with Master: 2a6c1e50223f43bdbd861350b1ab79d6 (127.27.170.194:43093)
09:04:55.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.117775 28331 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu as pid 28786
09:04:55.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.117964 28331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-1/wal/instance
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.122239 28331 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/wal
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/data
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/logs
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/data/info.pb
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.195:0
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.170.195
09:04:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.195
09:04:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:04:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.123787 28923 heartbeater.cc:344] Connected to a master server at 127.27.170.253:39067
09:04:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.123917 28923 heartbeater.cc:461] Registering TS with master...
09:04:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.124130 28921 heartbeater.cc:344] Connected to a master server at 127.27.170.252:40751
09:04:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.124256 28921 heartbeater.cc:461] Registering TS with master...
09:04:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.124392 28923 heartbeater.cc:507] Master 127.27.170.253:39067 requested a full tablet report, sending...
09:04:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.124744 28921 heartbeater.cc:507] Master 127.27.170.252:40751 requested a full tablet report, sending...
09:04:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.125526 28454 ts_manager.cc:194] Registered new tserver with Master: 2a6c1e50223f43bdbd861350b1ab79d6 (127.27.170.194:43093)
09:04:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.125562 28534 ts_manager.cc:194] Registered new tserver with Master: 2a6c1e50223f43bdbd861350b1ab79d6 (127.27.170.194:43093)
09:04:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.126483 28534 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.170.194:48513
09:04:55.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.180476 28642 catalog_manager.cc:1269] Loaded cluster ID: 36dfe5a18b044fa391749e4f6be4af0c
09:04:55.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.180855 28642 catalog_manager.cc:1562] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a: loading cluster ID for follower catalog manager: success
09:04:55.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.184585 28642 catalog_manager.cc:1584] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a: acquiring CA information for follower catalog manager: success
09:04:55.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.186888 28642 catalog_manager.cc:1612] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:04:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.430649 28926 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.430932 28926 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.431053 28926 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:55.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.439108 28926 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:55.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.439245 28926 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:04:55.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.439311 28926 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:04:55.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.439361 28926 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:04:55.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.439460 28926 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.170.195
09:04:55.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.448764 28926 tablet_server_runner.cc:78] Tablet server non-default flags:
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/data
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/wal
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.195:0
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/data/info.pb
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.195
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.170.195
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/logs
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 08:43:22 UTC on 24a791456cd2
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9476
09:04:55.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:55.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.450425 28926 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:55.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.452108 28926 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:55.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.460453 28933 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:55.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.460446 28932 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:55.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.461970 28935 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:55.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.462868 28926 server_base.cc:1047] running on GCE node
09:04:55.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.463557 28926 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:55.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.464671 28926 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.465932 28926 hybrid_clock.cc:648] HybridClock initialized: now 1765530295465861 us; error 42 us; skew 500 ppm
09:04:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.466277 28926 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:55.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.468931 28926 webserver.cc:492] Webserver started at http://127.27.170.195:33159/ using document root <none> and password file <none>
09:04:55.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.469830 28926 fs_manager.cc:362] Metadata directory not provided
09:04:55.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.469987 28926 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:55.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.470381 28926 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:04:55.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.473765 28926 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/data/instance:
09:04:55.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5c1306bdb9541f08cb6ed3b646d7895"
09:04:55.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:55 on dist-test-slave-hgx1"
09:04:55.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.474778 28926 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/wal/instance:
09:04:55.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5c1306bdb9541f08cb6ed3b646d7895"
09:04:55.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:55 on dist-test-slave-hgx1"
09:04:55.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.481393 28926 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.003s sys 0.003s
09:04:55.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.485546 28941 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.487185 28926 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.487342 28926 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/data,/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/wal
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5c1306bdb9541f08cb6ed3b646d7895"
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:55 on dist-test-slave-hgx1"
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.487571 28926 fs_report.cc:389] FS layout report
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/wal
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/wal
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/data/data
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:55.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:55.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:55.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:55.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:55.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:55.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:55.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:55.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.501617 28926 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.502364 28926 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.502627 28926 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.505713 28926 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:04:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.505818 28926 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:04:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.505931 28926 ts_tablet_manager.cc:616] Registered 0 tablets
09:04:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.506074 28926 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
09:04:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.585273 28926 rpc_server.cc:307] RPC server started. Bound to: 127.27.170.195:33963
09:04:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.585307 29063 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.170.195:33963 every 8 connection(s)
09:04:55.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.586562 28926 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/data/info.pb
09:04:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.588761 28331 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu as pid 28926
09:04:55.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.588929 28331 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/ts-2/wal/instance
09:04:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.589100 29065 heartbeater.cc:344] Connected to a master server at 127.27.170.254:35867
09:04:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.590688 29065 heartbeater.cc:461] Registering TS with master...
09:04:55.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.591563 29065 heartbeater.cc:507] Master 127.27.170.254:35867 requested a full tablet report, sending...
09:04:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.593268 28372 ts_manager.cc:194] Registered new tserver with Master: d5c1306bdb9541f08cb6ed3b646d7895 (127.27.170.195:33963)
09:04:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.595872 29064 heartbeater.cc:344] Connected to a master server at 127.27.170.252:40751
09:04:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.595992 29064 heartbeater.cc:461] Registering TS with master...
09:04:55.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.596563 29064 heartbeater.cc:507] Master 127.27.170.252:40751 requested a full tablet report, sending...
09:04:55.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.597839 28534 ts_manager.cc:194] Registered new tserver with Master: d5c1306bdb9541f08cb6ed3b646d7895 (127.27.170.195:33963)
09:04:55.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.598485 29066 heartbeater.cc:344] Connected to a master server at 127.27.170.253:39067
09:04:55.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.598606 29066 heartbeater.cc:461] Registering TS with master...
09:04:55.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.598872 28534 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.170.195:54697
09:04:55.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.598999 29066 heartbeater.cc:507] Master 127.27.170.253:39067 requested a full tablet report, sending...
09:04:55.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.600203 28453 ts_manager.cc:194] Registered new tserver with Master: d5c1306bdb9541f08cb6ed3b646d7895 (127.27.170.195:33963)
09:04:55.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.612783 28331 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
09:04:55.614 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:04:55.614 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
09:04:55.615 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.170.254"
port: 35867
}
bound_http_address {
host: "127.27.170.254"
port: 46879
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.170.253"
port: 39067
}
bound_http_address {
host: "127.27.170.253"
port: 34387
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.170.252"
port: 40751
}
bound_http_address {
host: "127.27.170.252"
port: 35295
}
}
}
09:04:55.615 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:04:55.616 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.170.193"
port: 38707
}
bound_http_address {
host: "127.27.170.193"
port: 38651
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.170.194"
port: 43093
}
bound_http_address {
host: "127.27.170.194"
port: 34817
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.170.195"
port: 33963
}
bound_http_address {
host: "127.27.170.195"
port: 33159
}
}
}
09:04:55.616 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
09:04:55.618 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
09:04:55.619 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.170.254:35867(127.27.170.254:35867)] connecting to peer
09:04:55.620 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:04:55.621 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.170.252:40751(127.27.170.252:40751)] connecting to peer
09:04:55.621 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.170.254:35867(127.27.170.254:35867)] Successfully connected to peer
09:04:55.622 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.27.170.253:39067(127.27.170.253:39067)] Successfully connected to peer
09:04:55.624 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.27.170.254
09:04:55.624 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.27.170.253
09:04:55.627 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.170.252:40751(127.27.170.252:40751)] Successfully connected to peer
09:04:55.628 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.27.170.252
09:04:55.635 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x3a214ca0, L:/127.0.0.1:45950 - R:/127.27.170.253:39067] using SASL/PLAIN
09:04:55.642 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd1aebd00, L:/127.0.0.1:33300 - R:/127.27.170.254:35867] using SASL/PLAIN
09:04:55.643 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x12767e60, L:/127.0.0.1:33424 - R:/127.27.170.252:40751] using SASL/PLAIN
09:04:55.646 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:04:55.646 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=27), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [19ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: OK, [27ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, deferred=null) after lookup
09:04:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.648478 28534 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33424:
09:04:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
09:04:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:04:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
09:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.650640 28534 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:04:55.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.666225 28781 heartbeater.cc:499] Master 127.27.170.252:40751 was elected leader, sending a full tablet report...
09:04:55.674 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 0156ecb556cd473bb7229ec267537527
09:04:55.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.716012 28999 tablet_service.cc:1505] Processing CreateTablet for tablet a62bbbb6a9d94ef8aed73601dabdb657 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:04:55.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.716238 28710 tablet_service.cc:1505] Processing CreateTablet for tablet 36dd6635cace411296a81d892c733544 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:04:55.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.716006 28853 tablet_service.cc:1505] Processing CreateTablet for tablet 36dd6635cace411296a81d892c733544 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:04:55.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.717335 28853 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 36dd6635cace411296a81d892c733544. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:55.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.717366 28710 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 36dd6635cace411296a81d892c733544. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:55.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.717398 28999 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a62bbbb6a9d94ef8aed73601dabdb657. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:55.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.719919 28854 tablet_service.cc:1505] Processing CreateTablet for tablet a62bbbb6a9d94ef8aed73601dabdb657 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:04:55.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.720402 28854 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a62bbbb6a9d94ef8aed73601dabdb657. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:55.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.716009 28711 tablet_service.cc:1505] Processing CreateTablet for tablet a62bbbb6a9d94ef8aed73601dabdb657 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:04:55.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.724472 28711 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a62bbbb6a9d94ef8aed73601dabdb657. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:55.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.724411 28998 tablet_service.cc:1505] Processing CreateTablet for tablet 36dd6635cace411296a81d892c733544 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:04:55.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.724947 28998 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 36dd6635cace411296a81d892c733544. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:55.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.736493 29091 tablet_bootstrap.cc:492] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875: Bootstrap starting.
09:04:55.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.736603 29090 tablet_bootstrap.cc:492] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895: Bootstrap starting.
09:04:55.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.737490 29089 tablet_bootstrap.cc:492] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6: Bootstrap starting.
09:04:55.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.739415 29090 tablet_bootstrap.cc:654] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895: Neither blocks nor log segments found. Creating new log.
09:04:55.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.740072 29091 tablet_bootstrap.cc:654] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875: Neither blocks nor log segments found. Creating new log.
09:04:55.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.740649 29090 log.cc:826] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895: Log is configured to *not* fsync() on all Append() calls
09:04:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.740890 29089 tablet_bootstrap.cc:654] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6: Neither blocks nor log segments found. Creating new log.
09:04:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.741425 29091 log.cc:826] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875: Log is configured to *not* fsync() on all Append() calls
09:04:55.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.742013 29089 log.cc:826] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6: Log is configured to *not* fsync() on all Append() calls
09:04:55.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.743458 29090 tablet_bootstrap.cc:492] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895: No bootstrap required, opened a new log
09:04:55.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.743695 29090 ts_tablet_manager.cc:1403] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.003s
09:04:55.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.743922 29091 tablet_bootstrap.cc:492] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875: No bootstrap required, opened a new log
09:04:55.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.744211 29091 ts_tablet_manager.cc:1403] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875: Time spent bootstrapping tablet: real 0.008s user 0.004s sys 0.003s
09:04:55.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.744756 29089 tablet_bootstrap.cc:492] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6: No bootstrap required, opened a new log
09:04:55.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.745002 29089 ts_tablet_manager.cc:1403] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.005s
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.749476 29090 raft_consensus.cc:359] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } }
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.749814 29090 raft_consensus.cc:385] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.749890 29090 raft_consensus.cc:740] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d5c1306bdb9541f08cb6ed3b646d7895, State: Initialized, Role: FOLLOWER
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.750443 29090 consensus_queue.cc:260] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } }
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.750734 29091 raft_consensus.cc:359] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.751042 29091 raft_consensus.cc:385] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.751127 29091 raft_consensus.cc:740] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2cad8cbd220442daa5b0ca7df42e0875, State: Initialized, Role: FOLLOWER
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.751518 29089 raft_consensus.cc:359] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } }
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.751838 29089 raft_consensus.cc:385] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.751753 29091 consensus_queue.cc:260] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.751919 29089 raft_consensus.cc:740] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a6c1e50223f43bdbd861350b1ab79d6, State: Initialized, Role: FOLLOWER
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.752622 29089 consensus_queue.cc:260] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } }
09:04:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.756683 29091 ts_tablet_manager.cc:1434] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875: Time spent starting tablet: real 0.012s user 0.011s sys 0.001s
09:04:55.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.769573 29091 tablet_bootstrap.cc:492] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875: Bootstrap starting.
09:04:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.772061 29091 tablet_bootstrap.cc:654] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875: Neither blocks nor log segments found. Creating new log.
09:04:55.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.777081 29064 heartbeater.cc:499] Master 127.27.170.252:40751 was elected leader, sending a full tablet report...
09:04:55.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.777647 29090 ts_tablet_manager.cc:1434] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895: Time spent starting tablet: real 0.034s user 0.022s sys 0.011s
09:04:55.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.778609 28921 heartbeater.cc:499] Master 127.27.170.252:40751 was elected leader, sending a full tablet report...
09:04:55.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.779325 29091 tablet_bootstrap.cc:492] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875: No bootstrap required, opened a new log
09:04:55.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.779470 29091 ts_tablet_manager.cc:1403] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.000s
09:04:55.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.780625 29090 tablet_bootstrap.cc:492] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895: Bootstrap starting.
09:04:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.782287 29090 tablet_bootstrap.cc:654] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895: Neither blocks nor log segments found. Creating new log.
09:04:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.788429 29091 raft_consensus.cc:359] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } }
09:04:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.788609 29091 raft_consensus.cc:385] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.788705 29091 raft_consensus.cc:740] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2cad8cbd220442daa5b0ca7df42e0875, State: Initialized, Role: FOLLOWER
09:04:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.788951 29091 consensus_queue.cc:260] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } }
09:04:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.789438 29089 ts_tablet_manager.cc:1434] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6: Time spent starting tablet: real 0.044s user 0.016s sys 0.019s
09:04:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.792341 29089 tablet_bootstrap.cc:492] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6: Bootstrap starting.
09:04:55.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.795317 29091 ts_tablet_manager.cc:1434] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875: Time spent starting tablet: real 0.016s user 0.002s sys 0.000s
09:04:55.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.798656 29089 tablet_bootstrap.cc:654] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6: Neither blocks nor log segments found. Creating new log.
09:04:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.800580 29089 tablet_bootstrap.cc:492] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6: No bootstrap required, opened a new log
09:04:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.800721 29089 ts_tablet_manager.cc:1403] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.000s
09:04:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.800830 29090 tablet_bootstrap.cc:492] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895: No bootstrap required, opened a new log
09:04:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.800992 29090 ts_tablet_manager.cc:1403] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895: Time spent bootstrapping tablet: real 0.020s user 0.006s sys 0.013s
09:04:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.801399 29089 raft_consensus.cc:359] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.801544 29089 raft_consensus.cc:385] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.801626 29089 raft_consensus.cc:740] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a6c1e50223f43bdbd861350b1ab79d6, State: Initialized, Role: FOLLOWER
09:04:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.801700 29090 raft_consensus.cc:359] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.801862 29090 raft_consensus.cc:385] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.801925 29090 raft_consensus.cc:740] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d5c1306bdb9541f08cb6ed3b646d7895, State: Initialized, Role: FOLLOWER
09:04:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.801833 29089 consensus_queue.cc:260] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.802112 29090 consensus_queue.cc:260] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.802291 29089 ts_tablet_manager.cc:1434] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
09:04:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.802587 29090 ts_tablet_manager.cc:1434] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
09:04:55.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.815213 29096 raft_consensus.cc:493] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:55.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.815415 29096 raft_consensus.cc:515] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } }
09:04:55.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.817042 29096 leader_election.cc:290] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d5c1306bdb9541f08cb6ed3b646d7895 (127.27.170.195:33963), 2a6c1e50223f43bdbd861350b1ab79d6 (127.27.170.194:43093)
09:04:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.822988 29097 raft_consensus.cc:493] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.823239 29097 raft_consensus.cc:515] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.824016 29019 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "a62bbbb6a9d94ef8aed73601dabdb657" candidate_uuid: "2cad8cbd220442daa5b0ca7df42e0875" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" is_pre_election: true
09:04:55.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.824527 29019 raft_consensus.cc:2468] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2cad8cbd220442daa5b0ca7df42e0875 in term 0.
09:04:55.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.824497 28876 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "a62bbbb6a9d94ef8aed73601dabdb657" candidate_uuid: "2cad8cbd220442daa5b0ca7df42e0875" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" is_pre_election: true
09:04:55.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.824872 28876 raft_consensus.cc:2468] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2cad8cbd220442daa5b0ca7df42e0875 in term 0.
09:04:55.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.825166 28659 leader_election.cc:304] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2cad8cbd220442daa5b0ca7df42e0875, d5c1306bdb9541f08cb6ed3b646d7895; no voters:
09:04:55.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.825546 29096 raft_consensus.cc:2804] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:04:55.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.825702 29096 raft_consensus.cc:493] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:04:55.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.825799 29096 raft_consensus.cc:3060] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Advancing to term 1
09:04:55.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.826254 29097 leader_election.cc:290] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2cad8cbd220442daa5b0ca7df42e0875 (127.27.170.193:38707), d5c1306bdb9541f08cb6ed3b646d7895 (127.27.170.195:33963)
09:04:55.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.828145 29096 raft_consensus.cc:515] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } }
09:04:55.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.828668 29096 leader_election.cc:290] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [CANDIDATE]: Term 1 election: Requested vote from peers d5c1306bdb9541f08cb6ed3b646d7895 (127.27.170.195:33963), 2a6c1e50223f43bdbd861350b1ab79d6 (127.27.170.194:43093)
09:04:55.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.829030 29019 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "a62bbbb6a9d94ef8aed73601dabdb657" candidate_uuid: "2cad8cbd220442daa5b0ca7df42e0875" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5c1306bdb9541f08cb6ed3b646d7895"
09:04:55.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.829120 28876 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "a62bbbb6a9d94ef8aed73601dabdb657" candidate_uuid: "2cad8cbd220442daa5b0ca7df42e0875" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a6c1e50223f43bdbd861350b1ab79d6"
09:04:55.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.829221 29019 raft_consensus.cc:3060] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Advancing to term 1
09:04:55.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.829280 28876 raft_consensus.cc:3060] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Advancing to term 1
09:04:55.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.831287 28876 raft_consensus.cc:2468] T a62bbbb6a9d94ef8aed73601dabdb657 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2cad8cbd220442daa5b0ca7df42e0875 in term 1.
09:04:55.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.831719 28658 leader_election.cc:304] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a6c1e50223f43bdbd861350b1ab79d6, 2cad8cbd220442daa5b0ca7df42e0875; no voters:
09:04:55.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.831985 29096 raft_consensus.cc:2804] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [term 1 FOLLOWER]: Leader election won for term 1
09:04:55.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.832273 29019 raft_consensus.cc:2468] T a62bbbb6a9d94ef8aed73601dabdb657 P d5c1306bdb9541f08cb6ed3b646d7895 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2cad8cbd220442daa5b0ca7df42e0875 in term 1.
09:04:55.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.832561 29096 raft_consensus.cc:697] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [term 1 LEADER]: Becoming Leader. State: Replica: 2cad8cbd220442daa5b0ca7df42e0875, State: Running, Role: LEADER
09:04:55.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.833475 29096 consensus_queue.cc:237] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } }
09:04:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.838841 28736 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "36dd6635cace411296a81d892c733544" candidate_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2cad8cbd220442daa5b0ca7df42e0875" is_pre_election: true
09:04:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.839180 28736 raft_consensus.cc:2468] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2a6c1e50223f43bdbd861350b1ab79d6 in term 0.
09:04:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.839538 28534 catalog_manager.cc:5654] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2cad8cbd220442daa5b0ca7df42e0875 (127.27.170.193). New cstate: current_term: 1 leader_uuid: "2cad8cbd220442daa5b0ca7df42e0875" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } health_report { overall_health: HEALTHY } } }
09:04:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.840315 29067 tablet.cc:2378] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:04:55.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.842149 28804 leader_election.cc:304] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a6c1e50223f43bdbd861350b1ab79d6, 2cad8cbd220442daa5b0ca7df42e0875; no voters:
09:04:55.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.842541 29097 raft_consensus.cc:2804] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:04:55.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.842660 29097 raft_consensus.cc:493] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:04:55.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.842733 29097 raft_consensus.cc:3060] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Advancing to term 1
09:04:55.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.843047 29019 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "36dd6635cace411296a81d892c733544" candidate_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" is_pre_election: true
09:04:55.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.843241 29019 raft_consensus.cc:2468] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2a6c1e50223f43bdbd861350b1ab79d6 in term 0.
09:04:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.845144 29097 raft_consensus.cc:515] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.845847 29097 leader_election.cc:290] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [CANDIDATE]: Term 1 election: Requested vote from peers 2cad8cbd220442daa5b0ca7df42e0875 (127.27.170.193:38707), d5c1306bdb9541f08cb6ed3b646d7895 (127.27.170.195:33963)
09:04:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.846542 29019 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "36dd6635cace411296a81d892c733544" candidate_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5c1306bdb9541f08cb6ed3b646d7895"
09:04:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.846729 29019 raft_consensus.cc:3060] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Advancing to term 1
09:04:55.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.849107 29019 raft_consensus.cc:2468] T 36dd6635cace411296a81d892c733544 P d5c1306bdb9541f08cb6ed3b646d7895 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2a6c1e50223f43bdbd861350b1ab79d6 in term 1.
09:04:55.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.849747 28736 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "36dd6635cace411296a81d892c733544" candidate_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2cad8cbd220442daa5b0ca7df42e0875"
09:04:55.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.849910 28736 raft_consensus.cc:3060] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Advancing to term 1
09:04:55.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.850106 28803 leader_election.cc:304] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a6c1e50223f43bdbd861350b1ab79d6, d5c1306bdb9541f08cb6ed3b646d7895; no voters:
09:04:55.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.850435 29097 raft_consensus.cc:2804] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 1 FOLLOWER]: Leader election won for term 1
09:04:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.850875 29097 raft_consensus.cc:697] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [term 1 LEADER]: Becoming Leader. State: Replica: 2a6c1e50223f43bdbd861350b1ab79d6, State: Running, Role: LEADER
09:04:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.851397 29097 consensus_queue.cc:237] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.852159 28736 raft_consensus.cc:2468] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2a6c1e50223f43bdbd861350b1ab79d6 in term 1.
09:04:55.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.856221 28534 catalog_manager.cc:5654] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2a6c1e50223f43bdbd861350b1ab79d6 (127.27.170.194). New cstate: current_term: 1 leader_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } health_report { overall_health: UNKNOWN } } }
09:04:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.862353 28924 tablet.cc:2378] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.878840 28534 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33432:
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:04:55.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.879763 28534 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:04:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.890133 28710 tablet_service.cc:1505] Processing CreateTablet for tablet 2e996b3c071f4969a635d5d47c796fad (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=4d4d5e428e1947dd808e2877488ec767]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:04:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.891151 28710 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2e996b3c071f4969a635d5d47c796fad. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.890796 28854 tablet_service.cc:1505] Processing CreateTablet for tablet 2e996b3c071f4969a635d5d47c796fad (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=4d4d5e428e1947dd808e2877488ec767]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:04:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.891706 28854 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2e996b3c071f4969a635d5d47c796fad. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.892455 28999 tablet_service.cc:1505] Processing CreateTablet for tablet 2e996b3c071f4969a635d5d47c796fad (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=4d4d5e428e1947dd808e2877488ec767]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:04:55.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.893229 28999 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2e996b3c071f4969a635d5d47c796fad. 1 dirs total, 0 dirs full, 0 dirs failed
09:04:55.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.897076 29089 tablet_bootstrap.cc:492] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6: Bootstrap starting.
09:04:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.898347 29090 tablet_bootstrap.cc:492] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: Bootstrap starting.
09:04:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.898768 29089 tablet_bootstrap.cc:654] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6: Neither blocks nor log segments found. Creating new log.
09:04:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:55.900506 28784 tablet.cc:2378] T 36dd6635cace411296a81d892c733544 P 2cad8cbd220442daa5b0ca7df42e0875: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:04:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.900964 29089 tablet_bootstrap.cc:492] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6: No bootstrap required, opened a new log
09:04:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.901098 29089 ts_tablet_manager.cc:1403] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.001s
09:04:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.901623 29089 raft_consensus.cc:359] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.901772 29089 raft_consensus.cc:385] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.901818 29089 raft_consensus.cc:740] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a6c1e50223f43bdbd861350b1ab79d6, State: Initialized, Role: FOLLOWER
09:04:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.901935 29090 tablet_bootstrap.cc:654] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: Neither blocks nor log segments found. Creating new log.
09:04:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.902007 29089 consensus_queue.cc:260] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.902314 29091 tablet_bootstrap.cc:492] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875: Bootstrap starting.
09:04:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.902447 29089 ts_tablet_manager.cc:1434] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
09:04:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.903563 29097 tablet_replica.cc:442] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } }
09:04:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.903781 29097 tablet_replica.cc:445] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.903931 29091 tablet_bootstrap.cc:654] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875: Neither blocks nor log segments found. Creating new log.
09:04:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.903954 29090 tablet_bootstrap.cc:492] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: No bootstrap required, opened a new log
09:04:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.904142 29090 ts_tablet_manager.cc:1403] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.001s
09:04:55.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.904863 29090 raft_consensus.cc:359] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.905064 29090 raft_consensus.cc:385] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:55.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.905148 29090 raft_consensus.cc:740] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d5c1306bdb9541f08cb6ed3b646d7895, State: Initialized, Role: FOLLOWER
09:04:55.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.905506 29090 consensus_queue.cc:260] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.906028 29090 ts_tablet_manager.cc:1434] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
09:04:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.906054 29095 tablet_replica.cc:442] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } }
09:04:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.906322 29095 tablet_replica.cc:445] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:55.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.913017 29091 tablet_bootstrap.cc:492] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875: No bootstrap required, opened a new log
09:04:55.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.913148 29091 ts_tablet_manager.cc:1403] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.002s
09:04:55.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.913712 29091 raft_consensus.cc:359] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.913865 29091 raft_consensus.cc:385] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:04:55.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.913909 29091 raft_consensus.cc:740] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2cad8cbd220442daa5b0ca7df42e0875, State: Initialized, Role: FOLLOWER
09:04:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.914068 29091 consensus_queue.cc:260] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.914517 29091 ts_tablet_manager.cc:1434] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
09:04:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.914450 29096 tablet_replica.cc:442] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } }
09:04:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:55.915103 29096 tablet_replica.cc:445] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:56.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.158895 29095 raft_consensus.cc:493] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:56.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.159087 29095 raft_consensus.cc:515] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.160655 29095 leader_election.cc:290] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2cad8cbd220442daa5b0ca7df42e0875 (127.27.170.193:38707), 2a6c1e50223f43bdbd861350b1ab79d6 (127.27.170.194:43093)
09:04:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.166374 28736 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2e996b3c071f4969a635d5d47c796fad" candidate_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2cad8cbd220442daa5b0ca7df42e0875" is_pre_election: true
09:04:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.166441 28876 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2e996b3c071f4969a635d5d47c796fad" candidate_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" is_pre_election: true
09:04:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.166591 28736 raft_consensus.cc:2468] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d5c1306bdb9541f08cb6ed3b646d7895 in term 0.
09:04:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.166683 28876 raft_consensus.cc:2468] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d5c1306bdb9541f08cb6ed3b646d7895 in term 0.
09:04:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.167011 28945 leader_election.cc:304] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2cad8cbd220442daa5b0ca7df42e0875, d5c1306bdb9541f08cb6ed3b646d7895; no voters:
09:04:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.167404 29095 raft_consensus.cc:2804] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:04:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.167528 29095 raft_consensus.cc:493] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:04:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.167618 29095 raft_consensus.cc:3060] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [term 0 FOLLOWER]: Advancing to term 1
09:04:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.169467 29095 raft_consensus.cc:515] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.170017 29095 leader_election.cc:290] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [CANDIDATE]: Term 1 election: Requested vote from peers 2cad8cbd220442daa5b0ca7df42e0875 (127.27.170.193:38707), 2a6c1e50223f43bdbd861350b1ab79d6 (127.27.170.194:43093)
09:04:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.170456 28876 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2e996b3c071f4969a635d5d47c796fad" candidate_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a6c1e50223f43bdbd861350b1ab79d6"
09:04:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.170478 28736 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "2e996b3c071f4969a635d5d47c796fad" candidate_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2cad8cbd220442daa5b0ca7df42e0875"
09:04:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.170660 28736 raft_consensus.cc:3060] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875 [term 0 FOLLOWER]: Advancing to term 1
09:04:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.170660 28876 raft_consensus.cc:3060] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6 [term 0 FOLLOWER]: Advancing to term 1
09:04:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.173249 28876 raft_consensus.cc:2468] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d5c1306bdb9541f08cb6ed3b646d7895 in term 1.
09:04:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.173249 28736 raft_consensus.cc:2468] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d5c1306bdb9541f08cb6ed3b646d7895 in term 1.
09:04:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.173682 28945 leader_election.cc:304] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2cad8cbd220442daa5b0ca7df42e0875, d5c1306bdb9541f08cb6ed3b646d7895; no voters:
09:04:56.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.173990 29095 raft_consensus.cc:2804] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [term 1 FOLLOWER]: Leader election won for term 1
09:04:56.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.174335 29095 raft_consensus.cc:697] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [term 1 LEADER]: Becoming Leader. State: Replica: d5c1306bdb9541f08cb6ed3b646d7895, State: Running, Role: LEADER
09:04:56.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.174801 29095 consensus_queue.cc:237] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } }
09:04:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.178378 29123 tablet_replica.cc:442] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: TxnStatusTablet state changed. Reason: New leader d5c1306bdb9541f08cb6ed3b646d7895. Latest consensus state: current_term: 1 leader_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } }
09:04:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.178779 29123 tablet_replica.cc:445] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: This TxnStatusTablet replica's current role is: LEADER
09:04:56.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.179886 29125 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
09:04:56.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.180320 29125 txn_status_manager.cc:930] Loading transaction status metadata into memory...
09:04:56.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.180685 28533 catalog_manager.cc:5654] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 reported cstate change: term changed from 0 to 1, leader changed from <none> to d5c1306bdb9541f08cb6ed3b646d7895 (127.27.170.195). New cstate: current_term: 1 leader_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } health_report { overall_health: HEALTHY } } }
09:04:56.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.208720 28876 raft_consensus.cc:1275] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6 [term 1 FOLLOWER]: Refusing update from remote peer d5c1306bdb9541f08cb6ed3b646d7895: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:04:56.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.208838 28736 raft_consensus.cc:1275] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875 [term 1 FOLLOWER]: Refusing update from remote peer d5c1306bdb9541f08cb6ed3b646d7895: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:04:56.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.209645 29123 consensus_queue.cc:1048] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:56.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.209923 29095 consensus_queue.cc:1048] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:56.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.220184 29097 tablet_replica.cc:442] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6: TxnStatusTablet state changed. Reason: New leader d5c1306bdb9541f08cb6ed3b646d7895. Latest consensus state: current_term: 1 leader_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } }
09:04:56.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.220450 29097 tablet_replica.cc:445] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:56.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.241690 29097 tablet_replica.cc:442] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } }
09:04:56.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.241946 29097 tablet_replica.cc:445] T 2e996b3c071f4969a635d5d47c796fad P 2a6c1e50223f43bdbd861350b1ab79d6: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:56.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.244602 29096 tablet_replica.cc:442] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875: TxnStatusTablet state changed. Reason: New leader d5c1306bdb9541f08cb6ed3b646d7895. Latest consensus state: current_term: 1 leader_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } }
09:04:56.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.244849 29096 tablet_replica.cc:445] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:56.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.244961 29095 tablet_replica.cc:442] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } }
09:04:56.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.245184 29095 tablet_replica.cc:445] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: This TxnStatusTablet replica's current role is: LEADER
09:04:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.250600 29123 tablet_replica.cc:442] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } }
09:04:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.250828 29123 tablet_replica.cc:445] T 2e996b3c071f4969a635d5d47c796fad P d5c1306bdb9541f08cb6ed3b646d7895: This TxnStatusTablet replica's current role is: LEADER
09:04:56.253 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
09:04:56.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.253796 29130 mvcc.cc:204] Tried to move back new op lower bound from 7231612093261668352 to 7231612093135912960. Current Snapshot: MvccSnapshot[applied={T|T < 7231612093261668352}]
09:04:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.255156 29096 tablet_replica.cc:442] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 } } peers { permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 } } peers { permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 } } }
09:04:56.255 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 0156ecb556cd473bb7229ec267537527
09:04:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.255365 29096 tablet_replica.cc:445] T 2e996b3c071f4969a635d5d47c796fad P 2cad8cbd220442daa5b0ca7df42e0875: This TxnStatusTablet replica's current role is: FOLLOWER
09:04:56.257 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.170.252:40751
09:04:56.257 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
09:04:56.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.258621 28331 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu pid 28501
09:04:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.260154 28575 generic_service.cc:196] Checking for leaks (request via RPC)
09:04:56.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.306731 29096 consensus_queue.cc:1048] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" member_type: VOTER last_known_addr { host: "127.27.170.194" port: 43093 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:56.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.316819 29096 consensus_queue.cc:1048] T a62bbbb6a9d94ef8aed73601dabdb657 P 2cad8cbd220442daa5b0ca7df42e0875 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
09:04:56.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:56.359401 28658 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.170.194:43093, user_credentials={real_user=slave}} blocked reactor thread for 44885us
09:04:56.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.410624 28331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu with pid 28501
09:04:56.433 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.170.252:40751(127.27.170.252:40751)] handling channelInactive
09:04:56.433 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.252:40751(127.27.170.252:40751)] cleaning up while in state READY due to: connection closed
09:04:56.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:56.433954 28657 connection.cc:537] client connection to 127.27.170.252:40751 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:04:56.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:56.434536 29064 heartbeater.cc:646] Failed to heartbeat to 127.27.170.252:40751 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.27.170.252:40751 (error 108)
09:04:56.435 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:04:56.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:56.434847 28781 heartbeater.cc:646] Failed to heartbeat to 127.27.170.252:40751 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:04:56.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:56.435446 28921 heartbeater.cc:646] Failed to heartbeat to 127.27.170.252:40751 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.27.170.252:40751 (error 108)
09:04:56.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.458431 29097 consensus_queue.cc:1048] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" member_type: VOTER last_known_addr { host: "127.27.170.195" port: 33963 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.465375 29143 consensus_queue.cc:1048] T 36dd6635cace411296a81d892c733544 P 2a6c1e50223f43bdbd861350b1ab79d6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" member_type: VOTER last_known_addr { host: "127.27.170.193" port: 38707 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:04:56.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.487185 29147 raft_consensus.cc:493] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 8b405358c0e04b69ba905ce322ad556e)
09:04:56.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.487490 29147 raft_consensus.cc:515] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.489085 29147 leader_election.cc:290] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867), 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751)
09:04:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.489481 28383 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bf0667fb0e154a5f82650259c7e41f1a" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "e57ee419ab3c4ad2811a115333f3909f" is_pre_election: true
09:04:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:56.491303 28438 leader_election.cc:336] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751): Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.491447 28438 leader_election.cc:304] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bf0667fb0e154a5f82650259c7e41f1a; no voters: 8b405358c0e04b69ba905ce322ad556e, e57ee419ab3c4ad2811a115333f3909f
09:04:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.491753 29147 raft_consensus.cc:2749] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
09:04:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.711825 29150 raft_consensus.cc:493] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 8b405358c0e04b69ba905ce322ad556e)
09:04:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.712059 29150 raft_consensus.cc:515] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:56.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.713336 29150 leader_election.cc:290] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067), 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751)
09:04:56.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.713932 28464 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e57ee419ab3c4ad2811a115333f3909f" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "bf0667fb0e154a5f82650259c7e41f1a" is_pre_election: true
09:04:56.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.714179 28464 raft_consensus.cc:2468] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e57ee419ab3c4ad2811a115333f3909f in term 1.
09:04:56.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.714856 28357 leader_election.cc:304] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bf0667fb0e154a5f82650259c7e41f1a, e57ee419ab3c4ad2811a115333f3909f; no voters:
09:04:56.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.715185 29150 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 1 FOLLOWER]: Leader pre-election won for term 2
09:04:56.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.715299 29150 raft_consensus.cc:493] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 1 FOLLOWER]: Starting leader election (detected failure of leader 8b405358c0e04b69ba905ce322ad556e)
09:04:56.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.715406 29150 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 1 FOLLOWER]: Advancing to term 2
09:04:56.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:56.715601 28357 leader_election.cc:336] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751): Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:56.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.717379 29150 raft_consensus.cc:515] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:56.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.717913 29150 leader_election.cc:290] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 2 election: Requested vote from peers bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067), 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751)
09:04:56.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.718314 28464 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e57ee419ab3c4ad2811a115333f3909f" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "bf0667fb0e154a5f82650259c7e41f1a"
09:04:56.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.718541 28464 raft_consensus.cc:3060] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 1 FOLLOWER]: Advancing to term 2
09:04:56.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:56.719122 28357 leader_election.cc:336] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751): Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111)
09:04:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.720543 28464 raft_consensus.cc:2468] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e57ee419ab3c4ad2811a115333f3909f in term 2.
09:04:56.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.720902 28357 leader_election.cc:304] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: bf0667fb0e154a5f82650259c7e41f1a, e57ee419ab3c4ad2811a115333f3909f; no voters: 8b405358c0e04b69ba905ce322ad556e
09:04:56.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.721149 29150 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 2 FOLLOWER]: Leader election won for term 2
09:04:56.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.721561 29150 raft_consensus.cc:697] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 2 LEADER]: Becoming Leader. State: Replica: e57ee419ab3c4ad2811a115333f3909f, State: Running, Role: LEADER
09:04:56.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.721980 29150 consensus_queue.cc:237] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:56.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.723305 29153 sys_catalog.cc:455] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: SysCatalogTable state changed. Reason: New leader e57ee419ab3c4ad2811a115333f3909f. Latest consensus state: current_term: 2 leader_uuid: "e57ee419ab3c4ad2811a115333f3909f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:56.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.723621 29153 sys_catalog.cc:458] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: This master's current role is: LEADER
09:04:56.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.724225 29154 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:04:56.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.726465 29154 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527]
09:04:56.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.726934 29154 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=4d4d5e428e1947dd808e2877488ec767]
09:04:56.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.728462 29154 tablet_loader.cc:96] loaded metadata for tablet 2e996b3c071f4969a635d5d47c796fad (table kudu_system.kudu_transactions [id=4d4d5e428e1947dd808e2877488ec767])
09:04:56.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.728884 29154 tablet_loader.cc:96] loaded metadata for tablet 36dd6635cace411296a81d892c733544 (table txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527])
09:04:56.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.729209 29154 tablet_loader.cc:96] loaded metadata for tablet a62bbbb6a9d94ef8aed73601dabdb657 (table txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527])
09:04:56.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.729462 29154 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:04:56.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.730242 29154 catalog_manager.cc:1269] Loaded cluster ID: 36dfe5a18b044fa391749e4f6be4af0c
09:04:56.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.730329 29154 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:04:56.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.731405 29154 catalog_manager.cc:1514] Loading token signing keys...
09:04:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.732265 29154 catalog_manager.cc:6038] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: Loaded TSK: 0
09:04:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.733057 29154 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:04:56.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.252:40751(127.27.170.252:40751)] connecting to peer
09:04:56.761 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.170.252:40751(127.27.170.252:40751): Connection refused: /127.27.170.252:40751
09:04:56.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.170.252:40751(127.27.170.252:40751)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.252:40751(127.27.170.252:40751): Connection refused: /127.27.170.252:40751
09:04:56.761 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.27.170.252:40751(127.27.170.252:40751) for tablet Kudu Master: Failed to connect to peer master-127.27.170.252:40751(127.27.170.252:40751): Connection refused: /127.27.170.252:40751
09:04:56.780 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.252:40751(127.27.170.252:40751)] connecting to peer
09:04:56.781 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.170.252:40751(127.27.170.252:40751): Connection refused: /127.27.170.252:40751
09:04:56.781 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.252:40751(127.27.170.252:40751)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.252:40751(127.27.170.252:40751): Connection refused: /127.27.170.252:40751
09:04:56.781 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.252:40751: Failed to connect to peer master-127.27.170.252:40751(127.27.170.252:40751): Connection refused: /127.27.170.252:40751
09:04:56.783 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:04:56.783 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=23), Traces: [0ms] sending RPC to server master-127.27.170.252:40751, [1ms] received response from server master-127.27.170.252:40751: Network error: Failed to connect to peer master-127.27.170.252:40751(127.27.170.252:40751): Connection refused: /127.27.170.252:40751, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.27.170.252:40751(127.27.170.252:40751): Connection refused: /127.27.170.252:40751, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [21ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: Network error: Failed to connect to peer master-127.27.170.252:40751(127.27.170.252:40751): Connection refused: /127.27.170.252:40751, [21ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, deferred=Deferred@1934473400(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@7f3f52b8, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@1559318f)) after lookup
09:04:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.828810 28464 raft_consensus.cc:1275] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 2 FOLLOWER]: Refusing update from remote peer e57ee419ab3c4ad2811a115333f3909f: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
09:04:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.829610 29153 consensus_queue.cc:1048] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [LEADER]: Connected to new peer: Peer: permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
09:04:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.831578 29147 sys_catalog.cc:455] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: SysCatalogTable state changed. Reason: New leader e57ee419ab3c4ad2811a115333f3909f. Latest consensus state: current_term: 2 leader_uuid: "e57ee419ab3c4ad2811a115333f3909f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.831828 29147 sys_catalog.cc:458] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: This master's current role is: FOLLOWER
09:04:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.834508 29150 sys_catalog.cc:455] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "e57ee419ab3c4ad2811a115333f3909f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.834785 29150 sys_catalog.cc:458] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: This master's current role is: LEADER
09:04:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.836062 29147 sys_catalog.cc:455] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "e57ee419ab3c4ad2811a115333f3909f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.836265 29147 sys_catalog.cc:458] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: This master's current role is: FOLLOWER
09:04:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:56.837215 28357 consensus_peers.cc:597] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f -> Peer 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751): Couldn't send request to peer 8b405358c0e04b69ba905ce322ad556e. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:56.840224 28373 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33326:
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:04:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:56.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:04:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:56.842216 28373 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:04:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:57.267930 28357 consensus_peers.cc:597] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f -> Peer 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751): Couldn't send request to peer 8b405358c0e04b69ba905ce322ad556e. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:04:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:57.749668 28357 consensus_peers.cc:597] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f -> Peer 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751): Couldn't send request to peer 8b405358c0e04b69ba905ce322ad556e. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:04:57.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:57.828148 28782 heartbeater.cc:499] Master 127.27.170.254:35867 was elected leader, sending a full tablet report...
09:04:57.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:57.962152 28357 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111) [suppressed 21 similar messages]
09:04:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.012259 28922 heartbeater.cc:499] Master 127.27.170.254:35867 was elected leader, sending a full tablet report...
09:04:58.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.247066 28357 consensus_peers.cc:597] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f -> Peer 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751): Couldn't send request to peer 8b405358c0e04b69ba905ce322ad556e. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:04:58.440 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a62bbbb6a9d94ef8aed73601dabdb657 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
09:04:58.440 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 36dd6635cace411296a81d892c733544 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
09:04:58.441 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=a62bbbb6a9d94ef8aed73601dabdb657@[2a6c1e50223f43bdbd861350b1ab79d6(127.27.170.194:43093),2cad8cbd220442daa5b0ca7df42e0875(127.27.170.193:38707)[L],d5c1306bdb9541f08cb6ed3b646d7895(127.27.170.195:33963)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=36dd6635cace411296a81d892c733544@[2a6c1e50223f43bdbd861350b1ab79d6(127.27.170.194:43093)[L],2cad8cbd220442daa5b0ca7df42e0875(127.27.170.193:38707),d5c1306bdb9541f08cb6ed3b646d7895(127.27.170.195:33963)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
09:04:58.441 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.27.170.254:35867, [2ms] Sub RPC GetTableLocations: received response from server master-127.27.170.254:35867: OK, deferred=null) after lookup
09:04:58.441 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 2cad8cbd220442daa5b0ca7df42e0875(127.27.170.193:38707)] connecting to peer
09:04:58.442 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 2cad8cbd220442daa5b0ca7df42e0875(127.27.170.193:38707)] Successfully connected to peer
09:04:58.455 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x3c4e2b4b, L:/127.0.0.1:33516 - R:/127.27.170.193:38707] using TOKEN/null
09:04:58.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.459152 28699 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.27.170.252:40751)
09:04:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.470095 29065 heartbeater.cc:499] Master 127.27.170.254:35867 was elected leader, sending a full tablet report...
09:04:58.493 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.170.252:40751
09:04:58.493 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
09:04:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.494648 28331 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:04:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal
09:04:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data
09:04:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/logs
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data/info.pb
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.252:40751
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.252
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35295
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33495/jwks.json
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:04:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:04:58.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.728093 28357 consensus_peers.cc:597] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f -> Peer 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751): Couldn't send request to peer 8b405358c0e04b69ba905ce322ad556e. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.252:40751: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
09:04:58.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.822568 29197 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:04:58.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.822875 29197 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:58.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.822978 29197 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:04:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.831318 29197 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:04:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.831437 29197 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:04:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.831516 29197 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:04:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.831571 29197 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:04:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.831638 29197 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:04:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.831679 29197 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:04:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.831732 29197 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33495/jwks.json
09:04:58.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.841676 29197 master_runner.cc:386] Master server non-default flags:
09:04:58.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:58.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.252:40751
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33495/jwks.json
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data/info.pb
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.252
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35295
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/logs
09:04:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:04:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:04:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:04:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 08:43:22 UTC on 24a791456cd2
09:04:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9476
09:04:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:04:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.843552 29197 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:58.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.845274 29197 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:04:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.852936 29203 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.852977 29204 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:58.853906 29206 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.854185 29197 server_base.cc:1047] running on GCE node
09:04:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.854895 29197 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:58.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.856266 29197 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.857481 29197 hybrid_clock.cc:648] HybridClock initialized: now 1765530298857412 us; error 41 us; skew 500 ppm
09:04:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.857813 29197 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.859894 29197 webserver.cc:492] Webserver started at http://127.27.170.252:35295/ using document root <none> and password file <none>
09:04:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.860595 29197 fs_manager.cc:362] Metadata directory not provided
09:04:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.860729 29197 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:58.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.865666 29197 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.002s sys 0.003s
09:04:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.868613 29212 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:04:58.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.869833 29197 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
09:04:58.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.869992 29197 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data,/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal
09:04:58.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b405358c0e04b69ba905ce322ad556e"
09:04:58.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:53 on dist-test-slave-hgx1"
09:04:58.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.870671 29197 fs_report.cc:389] FS layout report
09:04:58.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:58.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal
09:04:58.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/wal
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data/data
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.895226 29197 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:04:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.896080 29197 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.896338 29197 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.918022 29197 rpc_server.cc:307] RPC server started. Bound to: 127.27.170.252:40751
09:04:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.918046 29273 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.170.252:40751 every 8 connection(s)
09:04:58.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.920133 29197 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-2/data/info.pb
09:04:58.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.929898 28331 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu as pid 29197
09:04:58.930 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:04:58.930 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.170.254:35867
09:04:58.930 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
09:04:58.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.930462 29275 sys_catalog.cc:263] Verifying existing consensus state
09:04:58.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.931258 28331 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu pid 28342
09:04:58.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.932361 28413 generic_service.cc:196] Checking for leaks (request via RPC)
09:04:58.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.935101 29275 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: Bootstrap starting.
09:04:58.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.952303 29275 log.cc:826] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: Log is configured to *not* fsync() on all Append() calls
09:04:58.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.965387 29275 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:04:58.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.965854 29275 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: Bootstrap complete.
09:04:58.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.971561 29275 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:58.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.972290 29275 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b405358c0e04b69ba905ce322ad556e, State: Initialized, Role: FOLLOWER
09:04:58.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.972877 29275 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.973697 29279 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.973976 29279 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: This master's current role is: FOLLOWER
09:04:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.974712 29275 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: configured and running, proceeding with master startup.
09:04:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.987280 29290 catalog_manager.cc:1269] Loaded cluster ID: 36dfe5a18b044fa391749e4f6be4af0c
09:04:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.987388 29290 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: loading cluster ID for follower catalog manager: success
09:04:58.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.991317 29290 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: acquiring CA information for follower catalog manager: success
09:04:58.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:58.994787 29290 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:04:59.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.022832 28921 heartbeater.cc:344] Connected to a master server at 127.27.170.252:40751
09:04:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.026268 29229 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" instance_seqno: 1765530295100699) as {username='slave'} at 127.27.170.194:42225; Asking this server to re-register.
09:04:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.027081 28921 heartbeater.cc:461] Registering TS with master...
09:04:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.028962 29229 ts_manager.cc:194] Registered new tserver with Master: 2a6c1e50223f43bdbd861350b1ab79d6 (127.27.170.194:43093)
09:04:59.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.063405 28331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu with pid 28342
09:04:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.065657 29239 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 1 FOLLOWER]: Advancing to term 2
09:04:59.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.073257 29297 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "e57ee419ab3c4ad2811a115333f3909f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:59.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.074095 29279 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: SysCatalogTable state changed. Reason: New leader e57ee419ab3c4ad2811a115333f3909f. Latest consensus state: current_term: 2 leader_uuid: "e57ee419ab3c4ad2811a115333f3909f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:59.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.074316 29279 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: This master's current role is: FOLLOWER
09:04:59.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.074482 29297 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: This master's current role is: FOLLOWER
09:04:59.089 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.170.254:35867(127.27.170.254:35867)] handling channelInactive
09:04:59.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.254:35867(127.27.170.254:35867)] cleaning up while in state READY due to: connection closed
09:04:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:59.089048 29213 connection.cc:537] server connection from 127.0.0.1:33876 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:04:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:59.090624 28439 connection.cc:537] server connection from 127.0.0.1:33056 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:04:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:59.091001 29291 negotiation.cc:337] Failed RPC negotiation. Trace:
09:04:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:04:58.981132 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.27.170.254:35867 (local address 127.0.0.1:33332)
09:04:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:04:58.981876 (+ 744us) negotiation.cc:107] Waiting for socket to connect
09:04:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:04:58.981912 (+ 36us) client_negotiation.cc:174] Beginning negotiation
09:04:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:04:58.982061 (+ 149us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
09:04:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:04:59.089584 (+107523us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.27.170.254:35867: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:04:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":521,"thread_start_us":347,"threads_started":1}
09:04:59.091 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:04:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.091387 29197 master_runner.cc:186] Error getting master registration for 127.27.170.254:35867: Network error: Client connection negotiation failed: client connection to 127.27.170.254:35867: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
09:04:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.095204 29197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:04:59.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.365252 29298 raft_consensus.cc:493] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 2 FOLLOWER]: Starting pre-election (detected failure of leader e57ee419ab3c4ad2811a115333f3909f)
09:04:59.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.365443 29298 raft_consensus.cc:515] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:59.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.366216 29298 leader_election.cc:290] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867), 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751)
09:04:59.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:59.367560 28441 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.170.254:35867: connect: Connection refused (error 111) [suppressed 5 similar messages]
09:04:59.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:59.369206 28441 leader_election.cc:336] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867): Network error: Client connection negotiation failed: client connection to 127.27.170.254:35867: connect: Connection refused (error 111)
09:04:59.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.371307 29297 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 2 FOLLOWER]: Starting pre-election (detected failure of leader e57ee419ab3c4ad2811a115333f3909f)
09:04:59.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.371554 29297 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:59.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.372783 29239 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bf0667fb0e154a5f82650259c7e41f1a" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "8b405358c0e04b69ba905ce322ad556e" is_pre_election: true
09:04:59.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.373145 29239 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bf0667fb0e154a5f82650259c7e41f1a in term 2.
09:04:59.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.373306 29297 leader_election.cc:290] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867), bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067)
09:04:59.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.373629 28438 leader_election.cc:304] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b405358c0e04b69ba905ce322ad556e, bf0667fb0e154a5f82650259c7e41f1a; no voters: e57ee419ab3c4ad2811a115333f3909f
09:04:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.373890 29298 raft_consensus.cc:2804] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 2 FOLLOWER]: Leader pre-election won for term 3
09:04:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.374020 29298 raft_consensus.cc:493] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 2 FOLLOWER]: Starting leader election (detected failure of leader e57ee419ab3c4ad2811a115333f3909f)
09:04:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.374112 29298 raft_consensus.cc:3060] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 2 FOLLOWER]: Advancing to term 3
09:04:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:59.374342 29216 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.170.254:35867: connect: Connection refused (error 111)
09:04:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:59.375782 29216 leader_election.cc:336] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867): Network error: Client connection negotiation failed: client connection to 127.27.170.254:35867: connect: Connection refused (error 111)
09:04:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.376715 29298 raft_consensus.cc:515] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.377324 29298 leader_election.cc:290] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [CANDIDATE]: Term 3 election: Requested vote from peers e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867), 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751)
09:04:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.377791 29239 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bf0667fb0e154a5f82650259c7e41f1a" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "8b405358c0e04b69ba905ce322ad556e"
09:04:59.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.378021 29239 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 2 FOLLOWER]: Advancing to term 3
09:04:59.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:59.380144 28441 leader_election.cc:336] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867): Network error: Client connection negotiation failed: client connection to 127.27.170.254:35867: connect: Connection refused (error 111)
09:04:59.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.380409 29239 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bf0667fb0e154a5f82650259c7e41f1a in term 3.
09:04:59.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.380798 28438 leader_election.cc:304] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b405358c0e04b69ba905ce322ad556e, bf0667fb0e154a5f82650259c7e41f1a; no voters: e57ee419ab3c4ad2811a115333f3909f
09:04:59.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.381088 29298 raft_consensus.cc:2804] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 3 FOLLOWER]: Leader election won for term 3
09:04:59.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.381456 29298 raft_consensus.cc:697] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [term 3 LEADER]: Becoming Leader. State: Replica: bf0667fb0e154a5f82650259c7e41f1a, State: Running, Role: LEADER
09:04:59.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.381987 29298 consensus_queue.cc:237] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:04:59.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.382069 28464 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b405358c0e04b69ba905ce322ad556e" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "bf0667fb0e154a5f82650259c7e41f1a" is_pre_election: true
09:04:59.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.382774 29213 leader_election.cc:304] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8b405358c0e04b69ba905ce322ad556e; no voters: bf0667fb0e154a5f82650259c7e41f1a, e57ee419ab3c4ad2811a115333f3909f
09:04:59.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.383091 29297 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 3 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
09:04:59.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.384007 29305 sys_catalog.cc:455] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: SysCatalogTable state changed. Reason: New leader bf0667fb0e154a5f82650259c7e41f1a. Latest consensus state: current_term: 3 leader_uuid: "bf0667fb0e154a5f82650259c7e41f1a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:59.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.384274 29305 sys_catalog.cc:458] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: This master's current role is: LEADER
09:04:59.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.384850 29307 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:04:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.387174 29307 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527]
09:04:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.387642 29307 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=4d4d5e428e1947dd808e2877488ec767]
09:04:59.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.389149 29307 tablet_loader.cc:96] loaded metadata for tablet 2e996b3c071f4969a635d5d47c796fad (table kudu_system.kudu_transactions [id=4d4d5e428e1947dd808e2877488ec767])
09:04:59.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.389575 29307 tablet_loader.cc:96] loaded metadata for tablet 36dd6635cace411296a81d892c733544 (table txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527])
09:04:59.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.389918 29307 tablet_loader.cc:96] loaded metadata for tablet a62bbbb6a9d94ef8aed73601dabdb657 (table txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527])
09:04:59.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.390164 29307 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:04:59.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.390942 29307 catalog_manager.cc:1269] Loaded cluster ID: 36dfe5a18b044fa391749e4f6be4af0c
09:04:59.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.391031 29307 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:04:59.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.392192 29307 catalog_manager.cc:1514] Loading token signing keys...
09:04:59.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.393064 29307 catalog_manager.cc:6038] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a: Loaded TSK: 0
09:04:59.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.393828 29307 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:04:59.439 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.254:35867(127.27.170.254:35867)] connecting to peer
09:04:59.440 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.170.254:35867(127.27.170.254:35867): Connection refused: /127.27.170.254:35867
09:04:59.440 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.254:35867(127.27.170.254:35867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.254:35867(127.27.170.254:35867): Connection refused: /127.27.170.254:35867
09:04:59.440 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.27.170.254:35867(127.27.170.254:35867) for tablet Kudu Master: Failed to connect to peer master-127.27.170.254:35867(127.27.170.254:35867): Connection refused: /127.27.170.254:35867
09:04:59.459 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.254:35867(127.27.170.254:35867)] connecting to peer
09:04:59.460 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.252:40751(127.27.170.252:40751)] connecting to peer
09:04:59.460 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.170.254:35867(127.27.170.254:35867): Connection refused: /127.27.170.254:35867
09:04:59.460 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.170.252:40751(127.27.170.252:40751)] Successfully connected to peer
09:04:59.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.170.254:35867(127.27.170.254:35867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.254:35867(127.27.170.254:35867): Connection refused: /127.27.170.254:35867
09:04:59.460 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.254:35867: Failed to connect to peer master-127.27.170.254:35867(127.27.170.254:35867): Connection refused: /127.27.170.254:35867
09:04:59.464 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:04:59.464 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=24), Traces: [0ms] sending RPC to server master-127.27.170.254:35867, [1ms] received response from server master-127.27.170.254:35867: Network error: Failed to connect to peer master-127.27.170.254:35867(127.27.170.254:35867): Connection refused: /127.27.170.254:35867, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.27.170.254:35867(127.27.170.254:35867): Connection refused: /127.27.170.254:35867, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [21ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: Network error: Failed to connect to peer master-127.27.170.254:35867(127.27.170.254:35867): Connection refused: /127.27.170.254:35867, [24ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: OK, deferred=Deferred@1399789314(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@4f02f52, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@448f9446)) after lookup
09:04:59.473 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x179f91af, L:/127.0.0.1:33894 - R:/127.27.170.252:40751] using TOKEN/null
09:04:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.478222 29064 heartbeater.cc:344] Connected to a master server at 127.27.170.252:40751
09:04:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.479547 29229 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" instance_seqno: 1765530295578578) as {username='slave'} at 127.27.170.195:54359; Asking this server to re-register.
09:04:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.480211 29064 heartbeater.cc:461] Registering TS with master...
09:04:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.481494 29229 ts_manager.cc:194] Registered new tserver with Master: d5c1306bdb9541f08cb6ed3b646d7895 (127.27.170.195:33963)
09:04:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.487440 29239 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 3 FOLLOWER]: Refusing update from remote peer bf0667fb0e154a5f82650259c7e41f1a: Log matching property violated. Preceding OpId in replica: term: 2 index: 12. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
09:04:59.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.488152 29305 consensus_queue.cc:1048] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [LEADER]: Connected to new peer: Peer: permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
09:04:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.489912 29297 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: SysCatalogTable state changed. Reason: New leader bf0667fb0e154a5f82650259c7e41f1a. Latest consensus state: current_term: 3 leader_uuid: "bf0667fb0e154a5f82650259c7e41f1a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.490113 29297 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: This master's current role is: FOLLOWER
09:04:59.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.493134 29297 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "bf0667fb0e154a5f82650259c7e41f1a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:59.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.493326 29297 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: This master's current role is: FOLLOWER
09:04:59.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:59.497310 28441 consensus_peers.cc:597] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a -> Peer e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867): Couldn't send request to peer e57ee419ab3c4ad2811a115333f3909f. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.254:35867: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:04:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.503886 29305 sys_catalog.cc:455] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "bf0667fb0e154a5f82650259c7e41f1a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:04:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.505494 29305 sys_catalog.cc:458] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a [sys.catalog]: This master's current role is: LEADER
09:04:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.509351 28453 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45992:
09:04:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:04:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:04:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:59.514219 28453 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:04:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.830749 28781 heartbeater.cc:344] Connected to a master server at 127.27.170.252:40751
09:04:59.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.832594 29229 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" instance_seqno: 1765530294636706) as {username='slave'} at 127.27.170.193:50345; Asking this server to re-register.
09:04:59.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.833304 28781 heartbeater.cc:461] Registering TS with master...
09:04:59.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:04:59.834446 29229 ts_manager.cc:194] Registered new tserver with Master: 2cad8cbd220442daa5b0ca7df42e0875 (127.27.170.193:38707)
09:04:59.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:04:59.996229 28441 consensus_peers.cc:597] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a -> Peer e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867): Couldn't send request to peer e57ee419ab3c4ad2811a115333f3909f. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.254:35867: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:05:00.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:00.485471 29066 heartbeater.cc:499] Master 127.27.170.253:39067 was elected leader, sending a full tablet report...
09:05:00.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:00.536465 28441 consensus_peers.cc:597] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a -> Peer e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867): Couldn't send request to peer e57ee419ab3c4ad2811a115333f3909f. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.254:35867: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:05:00.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:00.828421 28783 heartbeater.cc:499] Master 127.27.170.253:39067 was elected leader, sending a full tablet report...
09:05:01.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.017691 28923 heartbeater.cc:499] Master 127.27.170.253:39067 was elected leader, sending a full tablet report...
09:05:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.037452 28441 consensus_peers.cc:597] T 00000000000000000000000000000000 P bf0667fb0e154a5f82650259c7e41f1a -> Peer e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867): Couldn't send request to peer e57ee419ab3c4ad2811a115333f3909f. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.254:35867: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:05:01.094 [DEBUG - Time-limited test] (Connection.java:748) [peer 2a6c1e50223f43bdbd861350b1ab79d6(127.27.170.194:43093)] connecting to peer
09:05:01.095 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 2a6c1e50223f43bdbd861350b1ab79d6(127.27.170.194:43093)] Successfully connected to peer
09:05:01.106 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x43711021, L:/127.0.0.1:44352 - R:/127.27.170.194:43093] using TOKEN/null
09:05:01.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.115484 28837 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.27.170.252:40751)
09:05:01.146 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.170.254:35867
09:05:01.146 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
09:05:01.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.146763 28331 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/logs
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data/info.pb
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.254:35867
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.254
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46879
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:05:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33495/jwks.json
09:05:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:05:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:05:01.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.462899 29351 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:05:01.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.463191 29351 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:05:01.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.463265 29351 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:05:01.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.471446 29351 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:05:01.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.471567 29351 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:05:01.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.471638 29351 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:05:01.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.471693 29351 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:05:01.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.471760 29351 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:05:01.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.471820 29351 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:05:01.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.471874 29351 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33495/jwks.json
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.483093 29351 master_runner.cc:386] Master server non-default flags:
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.170.212:40639
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.170.254:35867
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33495/jwks.json
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data/info.pb
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.170.254
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46879
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/logs
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 08:43:22 UTC on 24a791456cd2
09:05:01.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9476
09:05:01.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
09:05:01.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.484663 29351 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:05:01.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.486341 29351 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:05:01.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.494311 29357 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:05:01.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.494611 29360 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:05:01.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.494330 29358 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:05:01.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.495162 29351 server_base.cc:1047] running on GCE node
09:05:01.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.495857 29351 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:05:01.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.497234 29351 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:05:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.498469 29351 hybrid_clock.cc:648] HybridClock initialized: now 1765530301498399 us; error 45 us; skew 500 ppm
09:05:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.498764 29351 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:05:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.501389 29351 webserver.cc:492] Webserver started at http://127.27.170.254:46879/ using document root <none> and password file <none>
09:05:01.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.502326 29351 fs_manager.cc:362] Metadata directory not provided
09:05:01.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.502465 29351 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:05:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.511009 29351 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.000s sys 0.004s
09:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.515034 29366 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:05:01.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.517014 29351 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.517203 29351 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data,/tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e57ee419ab3c4ad2811a115333f3909f"
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 09:04:52 on dist-test-slave-hgx1"
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.518173 29351 fs_report.cc:389] FS layout report
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/wal
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data/data
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:05:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:05:01.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.538590 29351 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:05:01.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.539624 29351 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:05:01.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.539924 29351 kserver.cc:163] Server-wide thread pool size limit: 3276
09:05:01.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.561223 29351 rpc_server.cc:307] RPC server started. Bound to: 127.27.170.254:35867
09:05:01.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.561241 29427 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.170.254:35867 every 8 connection(s)
09:05:01.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.562728 29351 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskWDBSYb/test-tmp/mini-kudu-cluster5697330922304432602/master-0/data/info.pb
09:05:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.571255 29428 sys_catalog.cc:263] Verifying existing consensus state
09:05:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.572472 28331 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu as pid 29351
09:05:01.572 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:05:01.573 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.170.253:39067
09:05:01.573 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
09:05:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.573412 28331 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu pid 28422
09:05:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.574537 28494 generic_service.cc:196] Checking for leaks (request via RPC)
09:05:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.575410 29428 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: Bootstrap starting.
09:05:01.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.595613 29428 log.cc:826] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: Log is configured to *not* fsync() on all Append() calls
09:05:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.613266 29428 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:05:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.613794 29428 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: Bootstrap complete.
09:05:01.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.619779 29428 raft_consensus.cc:359] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:05:01.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.620534 29428 raft_consensus.cc:740] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: e57ee419ab3c4ad2811a115333f3909f, State: Initialized, Role: FOLLOWER
09:05:01.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.621222 29428 consensus_queue.cc:260] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:05:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.622056 29433 sys_catalog.cc:455] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:05:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.622325 29433 sys_catalog.cc:458] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: This master's current role is: FOLLOWER
09:05:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.623145 29428 sys_catalog.cc:565] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: configured and running, proceeding with master startup.
09:05:01.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.633586 29444 catalog_manager.cc:1269] Loaded cluster ID: 36dfe5a18b044fa391749e4f6be4af0c
09:05:01.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.633893 29444 catalog_manager.cc:1562] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: loading cluster ID for follower catalog manager: success
09:05:01.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.638015 29444 catalog_manager.cc:1584] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: acquiring CA information for follower catalog manager: success
09:05:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.640586 29444 catalog_manager.cc:1612] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:05:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.718948 28331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu with pid 28422
09:05:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.718992 29393 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 2 FOLLOWER]: Advancing to term 3
09:05:01.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.728806 29433 sys_catalog.cc:455] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: SysCatalogTable state changed. Reason: New leader bf0667fb0e154a5f82650259c7e41f1a. Latest consensus state: current_term: 3 leader_uuid: "bf0667fb0e154a5f82650259c7e41f1a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:05:01.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.729075 29433 sys_catalog.cc:458] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: This master's current role is: FOLLOWER
09:05:01.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.729341 29451 sys_catalog.cc:455] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "bf0667fb0e154a5f82650259c7e41f1a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:05:01.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.729617 29451 sys_catalog.cc:458] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: This master's current role is: FOLLOWER
09:05:01.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.738902 29370 connection.cc:537] server connection from 127.0.0.1:33744 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:05:01.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.739607 28942 connection.cc:537] server connection from 127.0.0.1:56114 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:05:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.740296 29445 negotiation.cc:337] Failed RPC negotiation. Trace:
09:05:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:01.651814 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.27.170.253:39067 (local address 127.0.0.1:45996)
09:05:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:01.652319 (+ 505us) negotiation.cc:107] Waiting for socket to connect
09:05:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:01.652337 (+ 18us) client_negotiation.cc:174] Beginning negotiation
09:05:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:01.652427 (+ 90us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
09:05:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:01.738989 (+ 86562us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:05:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":405}
09:05:01.740 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.27.170.253:39067(127.27.170.253:39067)] handling channelInactive
09:05:01.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state READY due to: connection closed
09:05:01.741 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.27.170.253:39067(127.27.170.253:39067) for tablet Kudu Master: connection closed
09:05:01.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:01.741762 29215 connection.cc:537] server connection from 127.0.0.1:33886 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:05:01.742 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:05:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.741207 29351 master_runner.cc:186] Error getting master registration for 127.27.170.253:39067: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
09:05:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.746497 29351 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:05:01.759 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.254:35867(127.27.170.254:35867)] connecting to peer
09:05:01.760 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:01.760 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.170.254:35867(127.27.170.254:35867)] Successfully connected to peer
09:05:01.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.772 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x114aa966, L:/127.0.0.1:33762 - R:/127.27.170.254:35867] using TOKEN/null
09:05:01.774 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:01.779 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:01.780 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.780 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.781 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:01.799 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:01.800 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.800 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.800 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:01.819 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:01.820 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.821 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:01.822 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] sending RPC to server master-127.27.170.253:39067, [141ms] received response from server master-127.27.170.253:39067: Network error: connection closed, [142ms] delaying RPC due to: Network error: connection closed, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [162ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [165ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [175ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [175ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [181ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [182ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [201ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [201ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [221ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [222ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, deferred=Deferred@744018216(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@22042a88 -> (continuation of Deferred@915035680 after retry RPC after error@195960030) -> (continuation of Deferred@1628785938 after retry RPC after error@1657170898) -> (continuation of Deferred@102304508 after retry RPC after error@2108341773), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@369c1177 -> (continuation of Deferred@915035680 after retry RPC after error@195960030) -> (continuation of Deferred@1628785938 after retry RPC after error@1657170898) -> (continuation of Deferred@102304508 after retry RPC after error@2108341773)))
09:05:01.823 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] sending RPC to server master-127.27.170.253:39067, [141ms] received response from server master-127.27.170.253:39067: Network error: connection closed, [142ms] delaying RPC due to: Network error: connection closed, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [162ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [165ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [175ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [175ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [181ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [182ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [201ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [201ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [221ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [222ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, deferred=Deferred@744018216(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@22042a88 -> (continuation of Deferred@915035680 after retry RPC after error@195960030) -> (continuation of Deferred@1628785938 after retry RPC after error@1657170898) -> (continuation of Deferred@102304508 after retry RPC after error@2108341773), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@369c1177 -> (continuation of Deferred@915035680 after retry RPC after error@195960030) -> (continuation of Deferred@1628785938 after retry RPC after error@1657170898) -> (continuation of Deferred@102304508 after retry RPC after error@2108341773)))
09:05:01.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.835443 28782 heartbeater.cc:344] Connected to a master server at 127.27.170.254:35867
09:05:01.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.838835 29383 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2cad8cbd220442daa5b0ca7df42e0875" instance_seqno: 1765530294636706) as {username='slave'} at 127.27.170.193:47969; Asking this server to re-register.
09:05:01.840 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.840250 28782 heartbeater.cc:461] Registering TS with master...
09:05:01.841 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.841 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.841 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:01.842558 29382 ts_manager.cc:194] Registered new tserver with Master: 2cad8cbd220442daa5b0ca7df42e0875 (127.27.170.193:38707)
09:05:01.845 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:01.860 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:01.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:01.880 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:01.881 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.881 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:01.900 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:01.903 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.903 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.903 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.903 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:01.920 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:01.921 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.921 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.921 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.924 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:01.960 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:01.961 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.962 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:01.962 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:02.000 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:02.001 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.001 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.001 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:02.003 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [2ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [5ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [22ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [22ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [42ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [42ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [62ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [64ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [82ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [84ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [85ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [123ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [162ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [162ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, deferred=Deferred@966294340(state=PENDING, result=null, callback=(continuation of Deferred@1139840038 after retry RPC after error@1054685525) -> (continuation of Deferred@1434895965 after retry RPC after error@792643058) -> (continuation of Deferred@1755854231 after retry RPC after error@1063824966) -> (continuation of Deferred@1547645912 after retry RPC after error@100101368) -> (continuation of Deferred@744154460 after retry RPC after error@1446671375) -> (continuation of Deferred@1941847267 after retry RPC after error@1301290688), errback=(continuation of Deferred@1139840038 after retry RPC after error@1054685525) -> (continuation of Deferred@1434895965 after retry RPC after error@792643058) -> (continuation of Deferred@1755854231 after retry RPC after error@1063824966) -> (continuation of Deferred@1547645912 after retry RPC after error@100101368) -> (continuation of Deferred@744154460 after retry RPC after error@1446671375) -> (continuation of Deferred@1941847267 after retry RPC after error@1301290688)))
09:05:02.004 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [2ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [5ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [22ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [22ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [42ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [42ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [62ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [64ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [82ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [84ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [85ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [122ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [123ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [162ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [162ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, deferred=Deferred@966294340(state=PENDING, result=null, callback=(continuation of Deferred@1139840038 after retry RPC after error@1054685525) -> (continuation of Deferred@1434895965 after retry RPC after error@792643058) -> (continuation of Deferred@1755854231 after retry RPC after error@1063824966) -> (continuation of Deferred@1547645912 after retry RPC after error@100101368) -> (continuation of Deferred@744154460 after retry RPC after error@1446671375) -> (continuation of Deferred@1941847267 after retry RPC after error@1301290688), errback=(continuation of Deferred@1139840038 after retry RPC after error@1054685525) -> (continuation of Deferred@1434895965 after retry RPC after error@792643058) -> (continuation of Deferred@1755854231 after retry RPC after error@1063824966) -> (continuation of Deferred@1547645912 after retry RPC after error@100101368) -> (continuation of Deferred@744154460 after retry RPC after error@1446671375) -> (continuation of Deferred@1941847267 after retry RPC after error@1301290688)))
09:05:02.020 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:02.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.026036 29451 raft_consensus.cc:493] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader bf0667fb0e154a5f82650259c7e41f1a)
09:05:02.026 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.026 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.026286 29451 raft_consensus.cc:515] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:05:02.026 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.028394 29451 leader_election.cc:290] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067), 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751)
09:05:02.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.029295 29367 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111)
09:05:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.057978 29367 leader_election.cc:336] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067): Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111)
09:05:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.058619 29239 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e57ee419ab3c4ad2811a115333f3909f" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "8b405358c0e04b69ba905ce322ad556e" is_pre_election: true
09:05:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.058903 29239 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e57ee419ab3c4ad2811a115333f3909f in term 3.
09:05:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.059046 29456 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 3 FOLLOWER]: Starting pre-election (detected failure of leader bf0667fb0e154a5f82650259c7e41f1a)
09:05:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.059203 29456 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:05:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.059690 28922 heartbeater.cc:344] Connected to a master server at 127.27.170.254:35867
09:05:02.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.060633 29367 leader_election.cc:304] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8b405358c0e04b69ba905ce322ad556e, e57ee419ab3c4ad2811a115333f3909f; no voters: bf0667fb0e154a5f82650259c7e41f1a
09:05:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.061061 29451 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 3 FOLLOWER]: Leader pre-election won for term 4
09:05:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.061190 29451 raft_consensus.cc:493] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 3 FOLLOWER]: Starting leader election (detected failure of leader bf0667fb0e154a5f82650259c7e41f1a)
09:05:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.061288 29451 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 3 FOLLOWER]: Advancing to term 4
09:05:02.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.064731 29451 raft_consensus.cc:515] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:05:02.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.065603 29383 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2a6c1e50223f43bdbd861350b1ab79d6" instance_seqno: 1765530295100699) as {username='slave'} at 127.27.170.194:47471; Asking this server to re-register.
09:05:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.066424 29451 leader_election.cc:290] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 4 election: Requested vote from peers bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067), 8b405358c0e04b69ba905ce322ad556e (127.27.170.252:40751)
09:05:02.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.067668 28922 heartbeater.cc:461] Registering TS with master...
09:05:02.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.068570 29239 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e57ee419ab3c4ad2811a115333f3909f" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "8b405358c0e04b69ba905ce322ad556e"
09:05:02.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.068818 29239 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 3 FOLLOWER]: Advancing to term 4
09:05:02.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.069619 29383 ts_manager.cc:194] Registered new tserver with Master: 2a6c1e50223f43bdbd861350b1ab79d6 (127.27.170.194:43093)
09:05:02.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.071555 29239 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e57ee419ab3c4ad2811a115333f3909f in term 4.
09:05:02.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.072244 29367 leader_election.cc:304] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8b405358c0e04b69ba905ce322ad556e, e57ee419ab3c4ad2811a115333f3909f; no voters:
09:05:02.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.072540 29451 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 4 FOLLOWER]: Leader election won for term 4
09:05:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.073887 29451 raft_consensus.cc:697] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [term 4 LEADER]: Becoming Leader. State: Replica: e57ee419ab3c4ad2811a115333f3909f, State: Running, Role: LEADER
09:05:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.074378 29393 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b405358c0e04b69ba905ce322ad556e" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "e57ee419ab3c4ad2811a115333f3909f" is_pre_election: true
09:05:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.074450 29451 consensus_queue.cc:237] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 3.13, Last appended by leader: 13, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } }
09:05:02.076 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.076421 29433 sys_catalog.cc:455] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: SysCatalogTable state changed. Reason: New leader e57ee419ab3c4ad2811a115333f3909f. Latest consensus state: current_term: 4 leader_uuid: "e57ee419ab3c4ad2811a115333f3909f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:05:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.076500 29456 leader_election.cc:290] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers e57ee419ab3c4ad2811a115333f3909f (127.27.170.254:35867), bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067)
09:05:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.076601 29433 sys_catalog.cc:458] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: This master's current role is: LEADER
09:05:02.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.077644 29367 leader_election.cc:336] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067): Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111)
09:05:02.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.078667 29462 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:05:02.079 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:02.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.079475 29213 leader_election.cc:336] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067): Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111)
09:05:02.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.079617 29213 leader_election.cc:304] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8b405358c0e04b69ba905ce322ad556e; no voters: bf0667fb0e154a5f82650259c7e41f1a, e57ee419ab3c4ad2811a115333f3909f
09:05:02.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.080168 29456 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 4 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
09:05:02.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.080 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.081 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.081511 29462 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527]
09:05:02.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.082134 29462 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=4d4d5e428e1947dd808e2877488ec767]
09:05:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.083848 29462 tablet_loader.cc:96] loaded metadata for tablet 2e996b3c071f4969a635d5d47c796fad (table kudu_system.kudu_transactions [id=4d4d5e428e1947dd808e2877488ec767])
09:05:02.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.084312 29462 tablet_loader.cc:96] loaded metadata for tablet 36dd6635cace411296a81d892c733544 (table txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527])
09:05:02.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.084652 29462 tablet_loader.cc:96] loaded metadata for tablet a62bbbb6a9d94ef8aed73601dabdb657 (table txn_manager_fallback_rolling [id=0156ecb556cd473bb7229ec267537527])
09:05:02.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.084916 29462 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:05:02.085 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751; will retry
09:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.085958 29462 catalog_manager.cc:1269] Loaded cluster ID: 36dfe5a18b044fa391749e4f6be4af0c
09:05:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.086047 29462 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:05:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.087181 29462 catalog_manager.cc:1514] Loading token signing keys...
09:05:02.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.087994 29462 catalog_manager.cc:6038] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f: Loaded TSK: 0
09:05:02.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.088804 29462 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:05:02.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.27.170.253:39067(127.27.170.253:39067)] connecting to peer
09:05:02.100 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.100 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.253:39067(127.27.170.253:39067)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.100 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.170.253:39067: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067
09:05:02.101 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:05:02.102 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=3, TimeoutTracker(timeout=250, elapsed=82), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [6ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [7ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [57ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [63ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [66ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [66ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [82ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, deferred=Deferred@294869030(state=PENDING, result=null, callback=(continuation of Deferred@85255109 after retry RPC after error@1221369682) -> (continuation of Deferred@333344861 after retry RPC after error@403134568), errback=(continuation of Deferred@85255109 after retry RPC after error@1221369682) -> (continuation of Deferred@333344861 after retry RPC after error@403134568))) after lookup
09:05:02.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.146155 29383 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33784:
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:05:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:05:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.149755 29428 master.cc:464] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 2 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
09:05:02.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.152968 29239 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [term 4 FOLLOWER]: Refusing update from remote peer e57ee419ab3c4ad2811a115333f3909f: Log matching property violated. Preceding OpId in replica: term: 3 index: 13. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
09:05:02.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.153568 29433 consensus_queue.cc:1048] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
09:05:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.155372 29456 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: SysCatalogTable state changed. Reason: New leader e57ee419ab3c4ad2811a115333f3909f. Latest consensus state: current_term: 4 leader_uuid: "e57ee419ab3c4ad2811a115333f3909f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:05:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.155629 29456 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: This master's current role is: FOLLOWER
09:05:02.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.156991 29451 sys_catalog.cc:455] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "e57ee419ab3c4ad2811a115333f3909f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:05:02.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.157218 29451 sys_catalog.cc:458] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f [sys.catalog]: This master's current role is: LEADER
09:05:02.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.158390 29367 consensus_peers.cc:597] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f -> Peer bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067): Couldn't send request to peer bf0667fb0e154a5f82650259c7e41f1a. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:05:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.158797 29456 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "e57ee419ab3c4ad2811a115333f3909f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e57ee419ab3c4ad2811a115333f3909f" member_type: VOTER last_known_addr { host: "127.27.170.254" port: 35867 } } peers { permanent_uuid: "bf0667fb0e154a5f82650259c7e41f1a" member_type: VOTER last_known_addr { host: "127.27.170.253" port: 39067 } } peers { permanent_uuid: "8b405358c0e04b69ba905ce322ad556e" member_type: VOTER last_known_addr { host: "127.27.170.252" port: 40751 } } }
09:05:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.158988 29456 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b405358c0e04b69ba905ce322ad556e [sys.catalog]: This master's current role is: FOLLOWER
09:05:02.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.271011 29413 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:33762 (request call id 14) took 168 ms (client timeout 168 ms). Trace:
09:05:02.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.271153 29413 rpcz_store.cc:269] 1212 09:05:02.102613 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:05:02.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:02.102687 (+ 74us) service_pool.cc:225] Handling call
09:05:02.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:02.270934 (+168247us) inbound_call.cc:173] Queueing success response
09:05:02.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":521,"init.queue_time_us":60,"thread_start_us":994,"threads_started":4}
09:05:02.272 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=4, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [6ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [7ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [57ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [63ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [66ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [66ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [82ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [83ms] sending RPC to server master-127.27.170.254:35867, [83ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [253ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@294869030(state=PENDING, result=null, callback=(continuation of Deferred@85255109 after retry RPC after error@1221369682) -> (continuation of Deferred@333344861 after retry RPC after error@403134568) -> (continuation of Deferred@984586854 after retry RPC@372618350), errback=(continuation of Deferred@85255109 after retry RPC after error@1221369682) -> (continuation of Deferred@333344861 after retry RPC after error@403134568) -> (continuation of Deferred@984586854 after retry RPC@372618350)))
09:05:02.272 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=4, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [6ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [7ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [57ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [57ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [61ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [63ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [66ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [66ms] delaying RPC due to: Service unavailable: Master config (127.27.170.254:35867,127.27.170.253:39067,127.27.170.252:40751) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.254:35867, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.253:39067, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.170.252:40751, [81ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.253:39067: Network error: Failed to connect to peer master-127.27.170.253:39067(127.27.170.253:39067): Connection refused: /127.27.170.253:39067, [82ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.254:35867: OK, [83ms] sending RPC to server master-127.27.170.254:35867, [83ms] Sub RPC ConnectToMaster: received response from server master-127.27.170.252:40751: OK, [253ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@294869030(state=PENDING, result=null, callback=(continuation of Deferred@85255109 after retry RPC after error@1221369682) -> (continuation of Deferred@333344861 after retry RPC after error@403134568) -> (continuation of Deferred@984586854 after retry RPC@372618350), errback=(continuation of Deferred@85255109 after retry RPC after error@1221369682) -> (continuation of Deferred@333344861 after retry RPC after error@403134568) -> (continuation of Deferred@984586854 after retry RPC@372618350)))
09:05:02.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.487072 29065 heartbeater.cc:344] Connected to a master server at 127.27.170.254:35867
09:05:02.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.488345 29383 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "d5c1306bdb9541f08cb6ed3b646d7895" instance_seqno: 1765530295578578) as {username='slave'} at 127.27.170.195:55859; Asking this server to re-register.
09:05:02.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.488878 29065 heartbeater.cc:461] Registering TS with master...
09:05:02.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.489121 29065 heartbeater.cc:507] Master 127.27.170.254:35867 requested a full tablet report, sending...
09:05:02.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.490264 29383 ts_manager.cc:194] Registered new tserver with Master: d5c1306bdb9541f08cb6ed3b646d7895 (127.27.170.195:33963)
09:05:02.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.530998 29413 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:33762 (request call id 15) took 250 ms (client timeout 250 ms). Trace:
09:05:02.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.531113 29413 rpcz_store.cc:269] 1212 09:05:02.280788 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:05:02.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:02.280852 (+ 64us) service_pool.cc:225] Handling call
09:05:02.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:02.530925 (+250073us) inbound_call.cc:173] Queueing success response
09:05:02.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:05:02.532 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.27.170.254:35867, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1980330605(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@54efa0dc, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@12557c8d))
09:05:02.532 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.27.170.254:35867, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1980330605(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@54efa0dc, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@12557c8d))
09:05:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.633519 29367 consensus_peers.cc:597] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f -> Peer bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067): Couldn't send request to peer bf0667fb0e154a5f82650259c7e41f1a. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:05:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:02.773697 28948 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.048s from last keepalive heartbeat (effective timeout is 1.000s)
09:05:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.780242 28975 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.27.170.252:40751)
09:05:02.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.790694 29413 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:33762 (request call id 16) took 250 ms (client timeout 250 ms). Trace:
09:05:02.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.790830 29413 rpcz_store.cc:269] 1212 09:05:02.540533 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:05:02.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:02.540601 (+ 68us) service_pool.cc:225] Handling call
09:05:02.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:02.790632 (+250031us) inbound_call.cc:173] Queueing success response
09:05:02.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:05:02.792 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.27.170.254:35867, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@695745645(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@14dfb935, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@3d1f1232))
09:05:02.792 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.27.170.254:35867, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@695745645(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@14dfb935, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@3d1f1232))
09:05:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.796212 29478 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
09:05:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:02.796274 29479 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
09:05:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:03.050410 29413 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:33762 (request call id 17) took 250 ms (client timeout 250 ms). Trace:
09:05:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:03.050535 29413 rpcz_store.cc:269] 1212 09:05:02.800221 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:05:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:02.800276 (+ 55us) service_pool.cc:225] Handling call
09:05:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 09:05:03.050337 (+250061us) inbound_call.cc:173] Queueing success response
09:05:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:05:03.051 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.27.170.254:35867, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@77049864(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@6ce47fa3, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@51b651b))
09:05:03.051 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.27.170.254:35867, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@77049864(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/832458229@6ce47fa3, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/2132512823@51b651b))
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:03.162380 29383 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33794:
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:05:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:05:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:05:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:05:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:03.163095 29383 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:05:03.176 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: d5c1306bdb9541f08cb6ed3b646d7895 (127.27.170.195:33963): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1765530296 last_transition_timestamp: 1765530302
09:05:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:03.194715 29367 consensus_peers.cc:597] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f -> Peer bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067): Couldn't send request to peer bf0667fb0e154a5f82650259c7e41f1a. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:05:03.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:03.699203 29367 consensus_peers.cc:597] T 00000000000000000000000000000000 P e57ee419ab3c4ad2811a115333f3909f -> Peer bf0667fb0e154a5f82650259c7e41f1a (127.27.170.253:39067): Couldn't send request to peer bf0667fb0e154a5f82650259c7e41f1a. Status: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:05:03.747 [DEBUG - main] (Connection.java:575) [peer master-127.27.170.253:39067(127.27.170.253:39067)] disconnecting while in state TERMINATED
09:05:03.747 [DEBUG - main] (Connection.java:575) [peer master-127.27.170.254:35867(127.27.170.254:35867)] disconnecting while in state READY
09:05:03.747 [DEBUG - main] (Connection.java:575) [peer master-127.27.170.252:40751(127.27.170.252:40751)] disconnecting while in state READY
09:05:03.747 [DEBUG - main] (Connection.java:575) [peer 2cad8cbd220442daa5b0ca7df42e0875(127.27.170.193:38707)] disconnecting while in state READY
09:05:03.747 [DEBUG - main] (Connection.java:575) [peer 2a6c1e50223f43bdbd861350b1ab79d6(127.27.170.194:43093)] disconnecting while in state READY
09:05:03.748 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.170.254:35867(127.27.170.254:35867)] handling channelInactive
09:05:03.748 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:05:03.748 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.254:35867(127.27.170.254:35867)] cleaning up while in state READY due to: connection closed
09:05:03.749 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.170.252:40751(127.27.170.252:40751)] handling channelInactive
09:05:03.749 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 2cad8cbd220442daa5b0ca7df42e0875(127.27.170.193:38707)] handling channelInactive
09:05:03.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.170.252:40751(127.27.170.252:40751)] cleaning up while in state READY due to: connection closed
09:05:03.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2cad8cbd220442daa5b0ca7df42e0875(127.27.170.193:38707)] cleaning up while in state READY due to: connection closed
09:05:03.749 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 2a6c1e50223f43bdbd861350b1ab79d6(127.27.170.194:43093)] handling channelInactive
09:05:03.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2a6c1e50223f43bdbd861350b1ab79d6(127.27.170.194:43093)] cleaning up while in state READY due to: connection closed
09:05:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:03.750154 28331 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu pid 28625
09:05:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:03.757232 28776 generic_service.cc:196] Checking for leaks (request via RPC)
09:05:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:03.857504 28782 heartbeater.cc:499] Master 127.27.170.254:35867 was elected leader, sending a full tablet report...
09:05:03.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:03.860819 28331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu with pid 28625
09:05:03.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:03.875630 29216 connection.cc:537] server connection from 127.27.170.193:50345 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:05:03.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:03.875850 28331 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu pid 28786
09:05:03.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:03.882086 28916 generic_service.cc:196] Checking for leaks (request via RPC)
09:05:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:03.987970 28331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu with pid 28786
09:05:04.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:04.002624 28331 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu pid 28926
09:05:04.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:04.009123 29059 generic_service.cc:196] Checking for leaks (request via RPC)
09:05:04.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:04.100737 29197 master_runner.cc:186] Error getting master registration for 127.27.170.253:39067: Network error: Client connection negotiation failed: client connection to 127.27.170.253:39067: connect: Connection refused (error 111),
09:05:04.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:04.100977 29197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:05:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:04.130055 28331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu with pid 28926
09:05:04.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:04.148155 28331 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu pid 29351
09:05:04.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:04.154170 29423 generic_service.cc:196] Checking for leaks (request via RPC)
09:05:04.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:04.266450 28331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu with pid 29351
09:05:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 09:05:04.283104 29216 connection.cc:537] server connection from 127.0.0.1:33906 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:05:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:04.283321 28331 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu pid 29197
09:05:04.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:04.289022 29269 generic_service.cc:196] Checking for leaks (request via RPC)
09:05:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 09:05:04.395143 28331 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskWDBSYb/build/asan/bin/kudu with pid 29197
09:05:04.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-12T09:05:04Z chronyd exiting
09:05:04.541 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)