14:07:16.661 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:07:16.661 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task7ZMTbR/test-tmp
14:07:16.661 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
14:07:16.661 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task7ZMTbR/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
14:07:16.663 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:07:16.789 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:07:16.789 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:07:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.790647 18065 mini_oidc.cc:150] Starting JWKS server
14:07:16.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.792953 18065 webserver.cc:469] Webserver started at http://127.0.0.1:33855/ using document root <none> and password file <none>
14:07:16.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.793140 18065 mini_oidc.cc:186] Starting OIDC Discovery server
14:07:16.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.793413 18065 webserver.cc:469] Webserver started at http://127.0.0.1:34693/ using document root <none> and password file <none>
14:07:16.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-22T14:07:16Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:07:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-22T14:07:16Z Disabled control of system clock
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.811882 18065 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/logs
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data/info.pb
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.126:34533
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.126
14:07:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33855/jwks.json
14:07:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:07:16.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:07:16.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.932664 18076 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:16.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.932956 18076 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:16.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.933012 18076 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.936398 18076 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.936473 18076 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:07:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.936491 18076 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.936511 18076 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.936528 18076 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.936546 18076 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.936561 18076 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33855/jwks.json
14:07:16.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.940649 18076 master_runner.cc:386] Master server non-default flags:
14:07:16.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:16.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:16.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:16.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.126:34533
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33855/jwks.json
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data/info.pb
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.126
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18076
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/logs
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:13 UTC on 24a791456cd2
14:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6714
14:07:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.942081 18076 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.943467 18076 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:16.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.951355 18082 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:16.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.951361 18081 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:16.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:16.951355 18084 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:16.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.951499 18076 server_base.cc:1048] running on GCE node
14:07:16.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.952412 18076 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:16.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.953516 18076 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:16.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.954689 18076 hybrid_clock.cc:648] HybridClock initialized: now 1750601236954665 us; error 46 us; skew 500 ppm
14:07:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.957108 18076 webserver.cc:469] Webserver started at http://127.17.164.126:34697/ using document root <none> and password file <none>
14:07:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.957772 18076 fs_manager.cc:362] Metadata directory not provided
14:07:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.957827 18076 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:16.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.958017 18076 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:07:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.959857 18076 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data/instance:
14:07:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "68e4a681ed9d4cc6bb513c955989849f"
14:07:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:16 on dist-test-slave-77bb"
14:07:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.960420 18076 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal/instance:
14:07:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "68e4a681ed9d4cc6bb513c955989849f"
14:07:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:16 on dist-test-slave-77bb"
14:07:16.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.964504 18076 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.003s sys 0.000s
14:07:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.967010 18090 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.968242 18076 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.968364 18076 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data,/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "68e4a681ed9d4cc6bb513c955989849f"
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:16 on dist-test-slave-77bb"
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.968459 18076 fs_report.cc:389] FS layout report
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data/data
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:16.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.983880 18076 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:16.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.984716 18076 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:16.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.984895 18076 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.994791 18076 rpc_server.cc:307] RPC server started. Bound to: 127.17.164.126:34533
14:07:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.994802 18152 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.164.126:34533 every 8 connection(s)
14:07:16.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.995939 18076 server_base.cc:1180] Dumped server information to /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data/info.pb
14:07:16.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.999375 18153 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:16.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.999473 18065 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu as pid 18076
14:07:16.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:16.999627 18065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal/instance
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.001616 18065 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/wal
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/data
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/logs
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/data/info.pb
14:07:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.125:41689
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.125
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33855/jwks.json
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:07:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:07:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.004902 18153 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } has no permanent_uuid. Determining permanent_uuid...
14:07:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.017782 18153 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } has no permanent_uuid. Determining permanent_uuid...
14:07:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.018705 18093 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111)
14:07:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.019604 18153 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.164.125:41689: Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111)
14:07:17.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.068920 18153 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } attempt: 1
14:07:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.070827 18153 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.164.125:41689: Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111)
14:07:17.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.123616 18155 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:17.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.124017 18155 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:17.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.124081 18155 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:17.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.127668 18155 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:17.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.127763 18155 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:07:17.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.127780 18155 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:17.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.127800 18155 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:17.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.127820 18155 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:17.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.127838 18155 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:17.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.127853 18155 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33855/jwks.json
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.132090 18155 master_runner.cc:386] Master server non-default flags:
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/data
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/wal
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.125:41689
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33855/jwks.json
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/data/info.pb
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.125
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18155
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/logs
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:13 UTC on 24a791456cd2
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6714
14:07:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.133642 18155 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:17.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.135025 18155 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:17.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.139084 18153 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } attempt: 2
14:07:17.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.140895 18153 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.164.125:41689: Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111)
14:07:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.143023 18163 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.143062 18164 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.143023 18166 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.143289 18155 server_base.cc:1048] running on GCE node
14:07:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.143880 18155 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.145022 18155 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:17.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.146202 18155 hybrid_clock.cc:648] HybridClock initialized: now 1750601237146176 us; error 45 us; skew 500 ppm
14:07:17.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.148784 18155 webserver.cc:469] Webserver started at http://127.17.164.125:42603/ using document root <none> and password file <none>
14:07:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.149518 18155 fs_manager.cc:362] Metadata directory not provided
14:07:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.149588 18155 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.149823 18155 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:07:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.151772 18155 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/data/instance:
14:07:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3c2b262c9aae4117ae997f8710cd51ae"
14:07:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:17 on dist-test-slave-77bb"
14:07:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.152384 18155 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/wal/instance:
14:07:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3c2b262c9aae4117ae997f8710cd51ae"
14:07:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:17 on dist-test-slave-77bb"
14:07:17.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.156529 18155 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.003s sys 0.001s
14:07:17.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.159320 18172 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:17.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.160773 18155 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.160925 18155 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/data,/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/wal
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3c2b262c9aae4117ae997f8710cd51ae"
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:17 on dist-test-slave-77bb"
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.161072 18155 fs_report.cc:389] FS layout report
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/wal
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/wal
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/data/data
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:17.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.176179 18155 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.177104 18155 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.177333 18155 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.186781 18155 rpc_server.cc:307] RPC server started. Bound to: 127.17.164.125:41689
14:07:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.186789 18234 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.164.125:41689 every 8 connection(s)
14:07:17.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.188067 18155 server_base.cc:1180] Dumped server information to /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/data/info.pb
14:07:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.189496 18065 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu as pid 18155
14:07:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.189615 18065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-1/wal/instance
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.190307 18065 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/logs
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data/info.pb
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.124:33213
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.124
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33855/jwks.json
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:07:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.191247 18235 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:17.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.196702 18235 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } has no permanent_uuid. Determining permanent_uuid...
14:07:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.207877 18235 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } has no permanent_uuid. Determining permanent_uuid...
14:07:17.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.215124 18235 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } has no permanent_uuid. Determining permanent_uuid...
14:07:17.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.216053 18175 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111)
14:07:17.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.217072 18235 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.164.124:33213: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111)
14:07:17.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.232174 18153 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } attempt: 3
14:07:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.237638 18153 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } has no permanent_uuid. Determining permanent_uuid...
14:07:17.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.239084 18153 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.164.124:33213: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111)
14:07:17.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.266301 18235 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } attempt: 1
14:07:17.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.268321 18235 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.164.124:33213: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111)
14:07:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.270290 18153 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } attempt: 1
14:07:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.271768 18153 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.164.124:33213: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111)
14:07:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.326671 18237 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.327051 18237 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.327150 18237 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.332139 18237 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.332444 18237 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:07:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.332501 18237 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.332532 18237 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.332562 18237 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.332587 18237 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.332608 18237 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33855/jwks.json
14:07:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.336526 18235 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } attempt: 2
14:07:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.338447 18235 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.164.124:33213: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111)
14:07:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.338729 18237 master_runner.cc:386] Master server non-default flags:
14:07:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.124:33213
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33855/jwks.json
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data/info.pb
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.124
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18237
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/logs
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:13 UTC on 24a791456cd2
14:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6714
14:07:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.340497 18237 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.342439 18237 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:17.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.346979 18153 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } attempt: 2
14:07:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.348968 18153 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.164.124:33213: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111)
14:07:17.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.351910 18245 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:17.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.352075 18237 server_base.cc:1048] running on GCE node
14:07:17.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.352067 18246 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:17.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.352144 18248 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:17.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.353168 18237 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:17.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.354535 18237 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:17.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.355748 18237 hybrid_clock.cc:648] HybridClock initialized: now 1750601237355712 us; error 49 us; skew 500 ppm
14:07:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.358523 18237 webserver.cc:469] Webserver started at http://127.17.164.124:45341/ using document root <none> and password file <none>
14:07:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.359515 18237 fs_manager.cc:362] Metadata directory not provided
14:07:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.359716 18237 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:17.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.360051 18237 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:07:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.362725 18237 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data/instance:
14:07:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e81f7c39b084eb1863771945700e98e"
14:07:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:17 on dist-test-slave-77bb"
14:07:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.363658 18237 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal/instance:
14:07:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e81f7c39b084eb1863771945700e98e"
14:07:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:17 on dist-test-slave-77bb"
14:07:17.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.369325 18237 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
14:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.372582 18254 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.373966 18237 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.374119 18237 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data,/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e81f7c39b084eb1863771945700e98e"
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:17 on dist-test-slave-77bb"
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.374203 18237 fs_report.cc:389] FS layout report
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data/data
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.396978 18237 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.397660 18237 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.397830 18237 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:17.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.407012 18237 rpc_server.cc:307] RPC server started. Bound to: 127.17.164.124:33213
14:07:17.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.407052 18316 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.164.124:33213 every 8 connection(s)
14:07:17.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.408277 18237 server_base.cc:1180] Dumped server information to /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data/info.pb
14:07:17.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.409161 18065 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu as pid 18237
14:07:17.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.409305 18065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal/instance
14:07:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.412178 18317 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.417336 18317 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } has no permanent_uuid. Determining permanent_uuid...
14:07:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.426901 18317 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } has no permanent_uuid. Determining permanent_uuid...
14:07:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.429616 18235 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } attempt: 3
14:07:17.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.432322 18317 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } has no permanent_uuid. Determining permanent_uuid...
14:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.436779 18235 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: Bootstrap starting.
14:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.439004 18235 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: Neither blocks nor log segments found. Creating new log.
14:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.439641 18235 log.cc:826] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: Log is configured to *not* fsync() on all Append() calls
14:07:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.441581 18235 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: No bootstrap required, opened a new log
14:07:17.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.442963 18317 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: Bootstrap starting.
14:07:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.444392 18235 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.444697 18235 raft_consensus.cc:383] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.444834 18235 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3c2b262c9aae4117ae997f8710cd51ae, State: Initialized, Role: FOLLOWER
14:07:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.445394 18235 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.445858 18317 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: Neither blocks nor log segments found. Creating new log.
14:07:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.445919 18325 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.446033 18325 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: This master's current role is: FOLLOWER
14:07:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.446591 18235 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: configured and running, proceeding with master startup.
14:07:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.447149 18317 log.cc:826] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: Log is configured to *not* fsync() on all Append() calls
14:07:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.448200 18153 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } attempt: 3
14:07:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.449611 18317 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: No bootstrap required, opened a new log
14:07:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.452021 18337 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:07:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.452093 18337 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:07:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.450744 18155 master_runner.cc:186] Error getting master registration for 127.17.164.126:34533: OK, instance_id { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" instance_seqno: 1750601236992341 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:07:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.453284 18155 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.453328 18317 raft_consensus.cc:357] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.453547 18317 raft_consensus.cc:383] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.453608 18317 raft_consensus.cc:738] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8e81f7c39b084eb1863771945700e98e, State: Initialized, Role: FOLLOWER
14:07:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.454150 18317 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.455575 18317 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: configured and running, proceeding with master startup.
14:07:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.455881 18339 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.456194 18339 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: This master's current role is: FOLLOWER
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.457456 18153 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: Bootstrap starting.
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.459424 18325 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.459568 18325 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.460592 18325 leader_election.cc:290] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213)
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.459686 18237 master_runner.cc:186] Error getting master registration for 127.17.164.126:34533: OK, instance_id { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" instance_seqno: 1750601236992341 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.462169 18237 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.462128 18281 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c2b262c9aae4117ae997f8710cd51ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8e81f7c39b084eb1863771945700e98e" is_pre_election: true
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.462175 18350 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.462345 18350 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.462453 18281 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3c2b262c9aae4117ae997f8710cd51ae in term 0.
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.463012 18175 leader_election.cc:304] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3c2b262c9aae4117ae997f8710cd51ae, 8e81f7c39b084eb1863771945700e98e; no voters:
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.463496 18325 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 0 FOLLOWER]: Leader pre-election won for term 1
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.463572 18325 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:07:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.463610 18325 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 0 FOLLOWER]: Advancing to term 1
14:07:17.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.465054 18325 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.465298 18153 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: Neither blocks nor log segments found. Creating new log.
14:07:17.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.465704 18325 leader_election.cc:290] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 1 election: Requested vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213)
14:07:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.466418 18281 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c2b262c9aae4117ae997f8710cd51ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8e81f7c39b084eb1863771945700e98e"
14:07:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.466565 18281 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 0 FOLLOWER]: Advancing to term 1
14:07:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.467852 18281 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3c2b262c9aae4117ae997f8710cd51ae in term 1.
14:07:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.468401 18175 leader_election.cc:304] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3c2b262c9aae4117ae997f8710cd51ae, 8e81f7c39b084eb1863771945700e98e; no voters:
14:07:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.468688 18325 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 1 FOLLOWER]: Leader election won for term 1
14:07:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.469430 18153 log.cc:826] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: Log is configured to *not* fsync() on all Append() calls
14:07:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.469408 18325 raft_consensus.cc:695] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 1 LEADER]: Becoming Leader. State: Replica: 3c2b262c9aae4117ae997f8710cd51ae, State: Running, Role: LEADER
14:07:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.470270 18325 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.474239 18153 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: No bootstrap required, opened a new log
14:07:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.474956 18325 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3c2b262c9aae4117ae997f8710cd51ae. Latest consensus state: current_term: 1 leader_uuid: "3c2b262c9aae4117ae997f8710cd51ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.475262 18325 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: This master's current role is: LEADER
14:07:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.479173 18153 raft_consensus.cc:357] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.479532 18153 raft_consensus.cc:383] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.479676 18153 raft_consensus.cc:738] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 68e4a681ed9d4cc6bb513c955989849f, State: Initialized, Role: FOLLOWER
14:07:17.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.480273 18153 consensus_queue.cc:260] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.481695 18153 sys_catalog.cc:564] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: configured and running, proceeding with master startup.
14:07:17.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.481799 18117 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c2b262c9aae4117ae997f8710cd51ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "68e4a681ed9d4cc6bb513c955989849f" is_pre_election: true
14:07:17.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.482272 18117 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3c2b262c9aae4117ae997f8710cd51ae in term 0.
14:07:17.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.483304 18116 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c2b262c9aae4117ae997f8710cd51ae" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "68e4a681ed9d4cc6bb513c955989849f"
14:07:17.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.483472 18116 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 0 FOLLOWER]: Advancing to term 1
14:07:17.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.485278 18116 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3c2b262c9aae4117ae997f8710cd51ae in term 1.
14:07:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.486614 18096 tablet.cc:2378] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:07:17.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.488363 18356 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.491252 18356 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: This master's current role is: FOLLOWER
14:07:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.491726 18356 raft_consensus.cc:491] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:07:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.492089 18356 raft_consensus.cc:513] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.496400 18199 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "68e4a681ed9d4cc6bb513c955989849f" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3c2b262c9aae4117ae997f8710cd51ae" is_pre_election: true
14:07:17.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.498034 18356 leader_election.cc:290] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689), 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213)
14:07:17.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.498420 18281 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "68e4a681ed9d4cc6bb513c955989849f" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8e81f7c39b084eb1863771945700e98e" is_pre_election: true
14:07:17.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.498608 18281 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 68e4a681ed9d4cc6bb513c955989849f in term 1.
14:07:17.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.498625 18368 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:07:17.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.498692 18368 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:07:17.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.499224 18093 leader_election.cc:304] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 68e4a681ed9d4cc6bb513c955989849f, 8e81f7c39b084eb1863771945700e98e; no voters: 3c2b262c9aae4117ae997f8710cd51ae
14:07:17.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.500243 18356 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 1 FOLLOWER]: Leader pre-election won for term 2
14:07:17.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.500414 18356 raft_consensus.cc:491] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:07:17.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.500535 18356 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 1 FOLLOWER]: Advancing to term 2
14:07:17.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.502198 18356 raft_consensus.cc:513] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.503196 18199 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "68e4a681ed9d4cc6bb513c955989849f" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3c2b262c9aae4117ae997f8710cd51ae"
14:07:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.504220 18281 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "68e4a681ed9d4cc6bb513c955989849f" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8e81f7c39b084eb1863771945700e98e"
14:07:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.504401 18281 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 1 FOLLOWER]: Advancing to term 2
14:07:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.505584 18281 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 68e4a681ed9d4cc6bb513c955989849f in term 2.
14:07:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.506121 18093 leader_election.cc:304] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 68e4a681ed9d4cc6bb513c955989849f, 8e81f7c39b084eb1863771945700e98e; no voters: 3c2b262c9aae4117ae997f8710cd51ae
14:07:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.506619 18357 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 2 FOLLOWER]: Leader election won for term 2
14:07:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.506923 18357 raft_consensus.cc:695] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 2 LEADER]: Becoming Leader. State: Replica: 68e4a681ed9d4cc6bb513c955989849f, State: Running, Role: LEADER
14:07:17.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.507164 18356 leader_election.cc:290] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [CANDIDATE]: Term 2 election: Requested vote from peers 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689), 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213)
14:07:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.507318 18357 consensus_queue.cc:237] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.507912 18355 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:07:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.509105 18355 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:07:17.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.512693 18356 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 68e4a681ed9d4cc6bb513c955989849f. Latest consensus state: current_term: 2 leader_uuid: "68e4a681ed9d4cc6bb513c955989849f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:17.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.512845 18356 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: This master's current role is: LEADER
14:07:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.517195 18373 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:07:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.517778 18281 raft_consensus.cc:1238] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 2 FOLLOWER]: Rejecting Update request from peer 3c2b262c9aae4117ae997f8710cd51ae for earlier term 1. Current term is 2. Ops: []
14:07:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.518553 18373 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:07:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.518877 18116 raft_consensus.cc:1238] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 2 LEADER]: Rejecting Update request from peer 3c2b262c9aae4117ae997f8710cd51ae for earlier term 1. Current term is 2. Ops: []
14:07:17.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.520077 18352 consensus_queue.cc:1046] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 }, Status: INVALID_TERM, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:17.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.520390 18352 consensus_queue.cc:1230] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [LEADER]: Peer 68e4a681ed9d4cc6bb513c955989849f log is divergent from this leader: its last log entry 2.1 is not in this leader's log and it has not received anything from this leader yet. Falling back to committed index 0
14:07:17.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.520454 18352 consensus_queue.cc:1046] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 }, Status: INVALID_TERM, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:17.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.520596 18352 raft_consensus.cc:3053] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 1 LEADER]: Stepping down as leader of term 1
14:07:17.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.520634 18352 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 3c2b262c9aae4117ae997f8710cd51ae, State: Running, Role: LEADER
14:07:17.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.520802 18352 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 1.2, Last appended by leader: 2, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:17.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.521061 18352 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 1 FOLLOWER]: Advancing to term 2
14:07:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.525220 18199 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 2 FOLLOWER]: Refusing update from remote peer 68e4a681ed9d4cc6bb513c955989849f: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 2 index: 2. (term mismatch)
14:07:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.525328 18199 pending_rounds.cc:85] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: Aborting all ops after (but not including) 1
14:07:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.525393 18199 pending_rounds.cc:107] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: Aborting uncommitted WRITE_OP operation due to leader change: 1.2
14:07:17.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.527141 18355 catalog_manager.cc:1462] Initializing Kudu cluster ID failed; change in term detected: 1 vs 2: Aborted: Op aborted by new leader
14:07:17.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.529407 18281 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 2 FOLLOWER]: Refusing update from remote peer 68e4a681ed9d4cc6bb513c955989849f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 2. (index mismatch)
14:07:17.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.530308 18357 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:17.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.530555 18356 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:17.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.530701 18356 consensus_queue.cc:1230] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [LEADER]: Peer 3c2b262c9aae4117ae997f8710cd51ae log is divergent from this leader: its last log entry 1.1 is not in this leader's log and it has not received anything from this leader yet. Falling back to committed index 0
14:07:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.536276 18199 pending_rounds.cc:85] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: Aborting all ops after (but not including) 0
14:07:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.536226 18339 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 68e4a681ed9d4cc6bb513c955989849f. Latest consensus state: current_term: 2 leader_uuid: "68e4a681ed9d4cc6bb513c955989849f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.536362 18199 pending_rounds.cc:107] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: Aborting uncommitted NO_OP operation due to leader change: 1.1
14:07:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.536394 18339 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: This master's current role is: FOLLOWER
14:07:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.536410 18199 raft_consensus.cc:2887] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 2 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted by new leader
14:07:17.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.538513 18375 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: SysCatalogTable state changed. Reason: New leader 68e4a681ed9d4cc6bb513c955989849f. Latest consensus state: current_term: 2 leader_uuid: "68e4a681ed9d4cc6bb513c955989849f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:17.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.538679 18375 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: This master's current role is: FOLLOWER
14:07:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.540302 18356 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "68e4a681ed9d4cc6bb513c955989849f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.540438 18356 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: This master's current role is: LEADER
14:07:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.540552 18376 mvcc.cc:204] Tried to move back new op lower bound from 7170462668893450240 to 7170462668843847680. Current Snapshot: MvccSnapshot[applied={T|T < 7170462668893450240}]
14:07:17.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.542718 18375 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "68e4a681ed9d4cc6bb513c955989849f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:17.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.542881 18375 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: This master's current role is: FOLLOWER
14:07:17.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.549322 18370 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "68e4a681ed9d4cc6bb513c955989849f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:17.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.549530 18370 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: This master's current role is: LEADER
14:07:17.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.552048 18339 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "68e4a681ed9d4cc6bb513c955989849f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:17.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.552250 18339 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: This master's current role is: FOLLOWER
14:07:17.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.556396 18373 catalog_manager.cc:1349] Generated new cluster ID: 1900d7fbdc174b3c9fae2ae657ce432f
14:07:17.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.556524 18373 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:07:17.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.600085 18373 catalog_manager.cc:1372] Generated new certificate authority record
14:07:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.601938 18373 catalog_manager.cc:1506] Loading token signing keys...
14:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.622047 18373 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: Generated new TSK 0
14:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.623481 18373 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:07:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.644346 18065 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/wal
14:07:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/data
14:07:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/logs
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/data/info.pb
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.65:0
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.164.65
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.65
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:07:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:07:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:07:17.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.800307 18382 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:17.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.800602 18382 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:17.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.800668 18382 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.804344 18382 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.804456 18382 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:07:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.804474 18382 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:07:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.804491 18382 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:07:17.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.804536 18382 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.164.65
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.808797 18382 tablet_server_runner.cc:78] Tablet server non-default flags:
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/data
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/wal
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.65:0
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/data/info.pb
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.65
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18382
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.164.65
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/logs
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:13 UTC on 24a791456cd2
14:07:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6714
14:07:17.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.810639 18382 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:17.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.812219 18382 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.822861 18387 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.823063 18382 server_base.cc:1048] running on GCE node
14:07:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.822713 18390 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:17.822713 18388 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:17.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.823971 18382 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:17.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.824790 18382 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:17.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.825965 18382 hybrid_clock.cc:648] HybridClock initialized: now 1750601237825940 us; error 48 us; skew 500 ppm
14:07:17.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.828835 18382 webserver.cc:469] Webserver started at http://127.17.164.65:39487/ using document root <none> and password file <none>
14:07:17.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.829501 18382 fs_manager.cc:362] Metadata directory not provided
14:07:17.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.829567 18382 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:17.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.829789 18382 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:07:17.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.831482 18382 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/data/instance:
14:07:17.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20ee773143da465dbdafbd4e4afd6a7b"
14:07:17.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:17 on dist-test-slave-77bb"
14:07:17.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.831997 18382 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/wal/instance:
14:07:17.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20ee773143da465dbdafbd4e4afd6a7b"
14:07:17.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:17 on dist-test-slave-77bb"
14:07:17.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.838264 18382 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.003s sys 0.000s
14:07:17.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.840843 18396 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.842146 18382 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.842306 18382 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/data,/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/wal
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20ee773143da465dbdafbd4e4afd6a7b"
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:17 on dist-test-slave-77bb"
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.842427 18382 fs_report.cc:389] FS layout report
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/wal
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/wal
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/data/data
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:17.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.861624 18382 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:17.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.862406 18382 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:17.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.862625 18382 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.864425 18382 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:07:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.864482 18382 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:07:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.864543 18382 ts_tablet_manager.cc:610] Registered 0 tablets
14:07:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.864571 18382 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:07:17.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.876839 18382 rpc_server.cc:307] RPC server started. Bound to: 127.17.164.65:37241
14:07:17.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.877137 18515 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.164.65:37241 every 8 connection(s)
14:07:17.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.878625 18382 server_base.cc:1180] Dumped server information to /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/data/info.pb
14:07:17.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.881559 18520 heartbeater.cc:344] Connected to a master server at 127.17.164.126:34533
14:07:17.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.881788 18520 heartbeater.cc:461] Registering TS with master...
14:07:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.882537 18520 heartbeater.cc:507] Master 127.17.164.126:34533 requested a full tablet report, sending...
14:07:17.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.884469 18107 ts_manager.cc:194] Registered new tserver with Master: 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241)
14:07:17.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.886023 18107 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.164.65:32881
14:07:17.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.886683 18065 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu as pid 18382
14:07:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.886811 18065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-0/wal/instance
14:07:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.887823 18519 heartbeater.cc:344] Connected to a master server at 127.17.164.124:33213
14:07:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.887921 18519 heartbeater.cc:461] Registering TS with master...
14:07:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.888199 18519 heartbeater.cc:507] Master 127.17.164.124:33213 requested a full tablet report, sending...
14:07:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.889484 18267 ts_manager.cc:194] Registered new tserver with Master: 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241)
14:07:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.889715 18521 heartbeater.cc:344] Connected to a master server at 127.17.164.125:41689
14:07:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.889816 18521 heartbeater.cc:461] Registering TS with master...
14:07:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.889991 18521 heartbeater.cc:507] Master 127.17.164.125:41689 requested a full tablet report, sending...
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.891139 18065 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/wal
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/data
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/logs
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/data/info.pb
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.66:0
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.164.66
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.66
14:07:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:07:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:07:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:07:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:17.891175 18189 ts_manager.cc:194] Registered new tserver with Master: 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241)
14:07:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.016412 18525 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.016700 18525 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.016763 18525 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:18.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.020255 18525 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:18.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.020350 18525 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:07:18.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.020368 18525 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:07:18.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.020385 18525 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:07:18.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.020424 18525 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.164.66
14:07:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.024416 18525 tablet_server_runner.cc:78] Tablet server non-default flags:
14:07:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/data
14:07:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/wal
14:07:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.66:0
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/data/info.pb
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.66
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18525
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.164.66
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/logs
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:13 UTC on 24a791456cd2
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6714
14:07:18.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.025736 18525 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:18.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.027177 18525 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.036525 18533 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.036520 18530 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.036667 18525 server_base.cc:1048] running on GCE node
14:07:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.036577 18531 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:18.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.037389 18525 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:18.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.038000 18525 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:18.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.039186 18525 hybrid_clock.cc:648] HybridClock initialized: now 1750601238039161 us; error 45 us; skew 500 ppm
14:07:18.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.041330 18525 webserver.cc:469] Webserver started at http://127.17.164.66:32779/ using document root <none> and password file <none>
14:07:18.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.041982 18525 fs_manager.cc:362] Metadata directory not provided
14:07:18.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.042042 18525 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:18.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.042248 18525 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:07:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.044270 18525 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/data/instance:
14:07:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68"
14:07:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:18 on dist-test-slave-77bb"
14:07:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.044795 18525 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/wal/instance:
14:07:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68"
14:07:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:18 on dist-test-slave-77bb"
14:07:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.049077 18525 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.003s sys 0.000s
14:07:18.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.051990 18539 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.053426 18525 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:07:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.053550 18525 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/data,/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/wal
14:07:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68"
14:07:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:18 on dist-test-slave-77bb"
14:07:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.053653 18525 fs_report.cc:389] FS layout report
14:07:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/wal
14:07:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/wal
14:07:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/data/data
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:18.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.067543 18525 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:18.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.068506 18525 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:18.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.068696 18525 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:18.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.070680 18525 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:07:18.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.070742 18525 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:07:18.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.070814 18525 ts_tablet_manager.cc:610] Registered 0 tablets
14:07:18.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.070835 18525 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:07:18.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.084955 18525 rpc_server.cc:307] RPC server started. Bound to: 127.17.164.66:45229
14:07:18.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.086777 18525 server_base.cc:1180] Dumped server information to /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/data/info.pb
14:07:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.088981 18065 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu as pid 18525
14:07:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.089097 18065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-1/wal/instance
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.090210 18663 heartbeater.cc:344] Connected to a master server at 127.17.164.126:34533
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.090502 18663 heartbeater.cc:461] Registering TS with master...
14:07:18.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.091194 18663 heartbeater.cc:507] Master 127.17.164.126:34533 requested a full tablet report, sending...
14:07:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.092357 18107 ts_manager.cc:194] Registered new tserver with Master: a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229)
14:07:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.093026 18107 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.164.66:54315
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.093618 18065 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/wal
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/data
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/logs
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/data/info.pb
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.67:0
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.164.67
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.67
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:07:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:07:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:07:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.095053 18661 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.164.66:45229 every 8 connection(s)
14:07:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.095942 18664 heartbeater.cc:344] Connected to a master server at 127.17.164.125:41689
14:07:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.096033 18664 heartbeater.cc:461] Registering TS with master...
14:07:18.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.096282 18664 heartbeater.cc:507] Master 127.17.164.125:41689 requested a full tablet report, sending...
14:07:18.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.096894 18662 heartbeater.cc:344] Connected to a master server at 127.17.164.124:33213
14:07:18.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.096971 18662 heartbeater.cc:461] Registering TS with master...
14:07:18.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.097131 18188 ts_manager.cc:194] Registered new tserver with Master: a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229)
14:07:18.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.097532 18662 heartbeater.cc:507] Master 127.17.164.124:33213 requested a full tablet report, sending...
14:07:18.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.098460 18267 ts_manager.cc:194] Registered new tserver with Master: a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229)
14:07:18.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.225486 18667 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:18.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.225761 18667 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:18.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.225822 18667 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:18.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.229413 18667 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:18.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.229552 18667 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:07:18.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.229573 18667 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:07:18.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.229589 18667 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:07:18.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.229635 18667 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.164.67
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.233731 18667 tablet_server_runner.cc:78] Tablet server non-default flags:
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/data
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/wal
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.67:0
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/data/info.pb
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.67
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18667
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.164.67
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/logs
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:13 UTC on 24a791456cd2
14:07:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6714
14:07:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.235406 18667 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.236913 18667 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.246685 18672 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.246889 18673 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.247117 18667 server_base.cc:1048] running on GCE node
14:07:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.246685 18675 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.247785 18667 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:18.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.248616 18667 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.249809 18667 hybrid_clock.cc:648] HybridClock initialized: now 1750601238249793 us; error 35 us; skew 500 ppm
14:07:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.252364 18667 webserver.cc:469] Webserver started at http://127.17.164.67:44977/ using document root <none> and password file <none>
14:07:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.253083 18667 fs_manager.cc:362] Metadata directory not provided
14:07:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.253147 18667 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.253361 18667 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:07:18.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.255323 18667 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/data/instance:
14:07:18.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20377aa649e94659bf7c8827077fc035"
14:07:18.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:18 on dist-test-slave-77bb"
14:07:18.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.255847 18667 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/wal/instance:
14:07:18.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20377aa649e94659bf7c8827077fc035"
14:07:18.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:18 on dist-test-slave-77bb"
14:07:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.260476 18667 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.005s sys 0.003s
14:07:18.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.263341 18681 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.264854 18667 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.264993 18667 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/data,/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/wal
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20377aa649e94659bf7c8827077fc035"
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:18 on dist-test-slave-77bb"
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.265103 18667 fs_report.cc:389] FS layout report
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/wal
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/wal
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/data/data
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:18.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:18.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.288774 18667 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:18.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.289628 18667 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:18.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.289814 18667 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:18.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.291761 18667 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:07:18.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.291836 18667 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:07:18.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.291898 18667 ts_tablet_manager.cc:610] Registered 0 tablets
14:07:18.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.291918 18667 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.306398 18667 rpc_server.cc:307] RPC server started. Bound to: 127.17.164.67:37573
14:07:18.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.307876 18667 server_base.cc:1180] Dumped server information to /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/data/info.pb
14:07:18.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.311514 18803 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.164.67:37573 every 8 connection(s)
14:07:18.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.312436 18805 heartbeater.cc:344] Connected to a master server at 127.17.164.126:34533
14:07:18.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.312773 18805 heartbeater.cc:461] Registering TS with master...
14:07:18.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.313985 18805 heartbeater.cc:507] Master 127.17.164.126:34533 requested a full tablet report, sending...
14:07:18.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.314635 18065 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu as pid 18667
14:07:18.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.314774 18065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/ts-2/wal/instance
14:07:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.314985 18804 heartbeater.cc:344] Connected to a master server at 127.17.164.124:33213
14:07:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.314965 18107 ts_manager.cc:194] Registered new tserver with Master: 20377aa649e94659bf7c8827077fc035 (127.17.164.67:37573)
14:07:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.315062 18804 heartbeater.cc:461] Registering TS with master...
14:07:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.315245 18804 heartbeater.cc:507] Master 127.17.164.124:33213 requested a full tablet report, sending...
14:07:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.315644 18107 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.164.67:36651
14:07:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.315927 18267 ts_manager.cc:194] Registered new tserver with Master: 20377aa649e94659bf7c8827077fc035 (127.17.164.67:37573)
14:07:18.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.319064 18806 heartbeater.cc:344] Connected to a master server at 127.17.164.125:41689
14:07:18.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.319183 18806 heartbeater.cc:461] Registering TS with master...
14:07:18.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.319408 18806 heartbeater.cc:507] Master 127.17.164.125:41689 requested a full tablet report, sending...
14:07:18.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.320235 18189 ts_manager.cc:194] Registered new tserver with Master: 20377aa649e94659bf7c8827077fc035 (127.17.164.67:37573)
14:07:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.322510 18065 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
14:07:18.323 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:07:18.323 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:07:18.324 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.17.164.126"
port: 34533
}
bound_http_address {
host: "127.17.164.126"
port: 34697
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.17.164.125"
port: 41689
}
bound_http_address {
host: "127.17.164.125"
port: 42603
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.17.164.124"
port: 33213
}
bound_http_address {
host: "127.17.164.124"
port: 45341
}
}
}
14:07:18.324 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:07:18.325 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.17.164.65"
port: 37241
}
bound_http_address {
host: "127.17.164.65"
port: 39487
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.17.164.66"
port: 45229
}
bound_http_address {
host: "127.17.164.66"
port: 32779
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.17.164.67"
port: 37573
}
bound_http_address {
host: "127.17.164.67"
port: 44977
}
}
}
14:07:18.325 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:07:18.327 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:07:18.330 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.164.124:33213(127.17.164.124:33213)] connecting to peer
14:07:18.331 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:18.332 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.164.125:41689(127.17.164.125:41689)] connecting to peer
14:07:18.332 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.17.164.124:33213(127.17.164.124:33213)] Successfully connected to peer
14:07:18.333 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.17.164.126:34533(127.17.164.126:34533)] Successfully connected to peer
14:07:18.334 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.17.164.124
14:07:18.335 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.17.164.126
14:07:18.338 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.17.164.125:41689(127.17.164.125:41689)] Successfully connected to peer
14:07:18.347 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.17.164.125
14:07:18.348 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xcbce9282, L:/127.0.0.1:39246 - R:/127.17.164.126:34533] using SASL/PLAIN
14:07:18.350 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:07:18.351 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=22), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [21ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, deferred=null) after lookup
14:07:18.352 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xebe78a23, L:/127.0.0.1:57576 - R:/127.17.164.124:33213] using SASL/PLAIN
14:07:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.352481 18107 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39246:
14:07:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
14:07:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:07:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
14:07:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:07:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.356181 18107 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:07:18.361 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x92d6680f, L:/127.0.0.1:40438 - R:/127.17.164.125:41689] using SASL/PLAIN
14:07:18.369 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 93e6eaae358242959886acaf4bb50364
14:07:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.382881 18731 tablet_service.cc:1468] Processing CreateTablet for tablet 991f2c43aa654bb9a26fb5f15304e043 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:07:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.382881 18446 tablet_service.cc:1468] Processing CreateTablet for tablet 991f2c43aa654bb9a26fb5f15304e043 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:07:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.383919 18731 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 991f2c43aa654bb9a26fb5f15304e043. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.383917 18446 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 991f2c43aa654bb9a26fb5f15304e043. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:18.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.382882 18730 tablet_service.cc:1468] Processing CreateTablet for tablet 9b182fc8b3834a57a90c369bd9f582cb (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:07:18.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.386056 18730 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9b182fc8b3834a57a90c369bd9f582cb. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:18.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.382884 18445 tablet_service.cc:1468] Processing CreateTablet for tablet 9b182fc8b3834a57a90c369bd9f582cb (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:07:18.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.386708 18445 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9b182fc8b3834a57a90c369bd9f582cb. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.392309 18824 tablet_bootstrap.cc:492] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035: Bootstrap starting.
14:07:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.393854 18825 tablet_bootstrap.cc:492] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b: Bootstrap starting.
14:07:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.394703 18824 tablet_bootstrap.cc:654] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035: Neither blocks nor log segments found. Creating new log.
14:07:18.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.395421 18824 log.cc:826] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035: Log is configured to *not* fsync() on all Append() calls
14:07:18.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.396200 18825 tablet_bootstrap.cc:654] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b: Neither blocks nor log segments found. Creating new log.
14:07:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.397050 18824 tablet_bootstrap.cc:492] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035: No bootstrap required, opened a new log
14:07:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.397209 18824 ts_tablet_manager.cc:1397] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
14:07:18.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.397338 18825 log.cc:826] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b: Log is configured to *not* fsync() on all Append() calls
14:07:18.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.398801 18825 tablet_bootstrap.cc:492] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b: No bootstrap required, opened a new log
14:07:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.398989 18825 ts_tablet_manager.cc:1397] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.003s
14:07:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.399567 18824 raft_consensus.cc:357] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.399741 18824 raft_consensus.cc:383] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:18.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.399780 18824 raft_consensus.cc:738] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 20377aa649e94659bf7c8827077fc035, State: Initialized, Role: FOLLOWER
14:07:18.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.400159 18824 consensus_queue.cc:260] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.401293 18824 ts_tablet_manager.cc:1428] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
14:07:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.401646 18825 raft_consensus.cc:357] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } }
14:07:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.401846 18825 raft_consensus.cc:383] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.401844 18805 heartbeater.cc:499] Master 127.17.164.126:34533 was elected leader, sending a full tablet report...
14:07:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.401896 18825 raft_consensus.cc:738] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 20ee773143da465dbdafbd4e4afd6a7b, State: Initialized, Role: FOLLOWER
14:07:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.402382 18825 consensus_queue.cc:260] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } }
14:07:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.403173 18825 ts_tablet_manager.cc:1428] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b: Time spent starting tablet: real 0.004s user 0.004s sys 0.001s
14:07:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.404563 18520 heartbeater.cc:499] Master 127.17.164.126:34533 was elected leader, sending a full tablet report...
14:07:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.401841 18824 tablet_bootstrap.cc:492] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035: Bootstrap starting.
14:07:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.405840 18824 tablet_bootstrap.cc:654] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035: Neither blocks nor log segments found. Creating new log.
14:07:18.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.406548 18825 tablet_bootstrap.cc:492] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b: Bootstrap starting.
14:07:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.407546 18825 tablet_bootstrap.cc:654] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b: Neither blocks nor log segments found. Creating new log.
14:07:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.408444 18588 tablet_service.cc:1468] Processing CreateTablet for tablet 9b182fc8b3834a57a90c369bd9f582cb (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:07:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.408679 18825 tablet_bootstrap.cc:492] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b: No bootstrap required, opened a new log
14:07:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.408751 18825 ts_tablet_manager.cc:1397] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
14:07:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.409106 18588 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9b182fc8b3834a57a90c369bd9f582cb. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.409204 18825 raft_consensus.cc:357] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.409289 18825 raft_consensus.cc:383] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.409317 18825 raft_consensus.cc:738] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 20ee773143da465dbdafbd4e4afd6a7b, State: Initialized, Role: FOLLOWER
14:07:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.409456 18825 consensus_queue.cc:260] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.409794 18825 ts_tablet_manager.cc:1428] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b: Time spent starting tablet: real 0.001s user 0.001s sys 0.001s
14:07:18.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.407837 18589 tablet_service.cc:1468] Processing CreateTablet for tablet 991f2c43aa654bb9a26fb5f15304e043 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:07:18.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.413089 18589 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 991f2c43aa654bb9a26fb5f15304e043. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.417809 18835 tablet_bootstrap.cc:492] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68: Bootstrap starting.
14:07:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.418155 18824 tablet_bootstrap.cc:492] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035: No bootstrap required, opened a new log
14:07:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.418262 18824 ts_tablet_manager.cc:1397] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035: Time spent bootstrapping tablet: real 0.016s user 0.002s sys 0.000s
14:07:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.418303 18829 raft_consensus.cc:491] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:07:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.418466 18829 raft_consensus.cc:513] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.418762 18824 raft_consensus.cc:357] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } }
14:07:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.418869 18824 raft_consensus.cc:383] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.418902 18824 raft_consensus.cc:738] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 20377aa649e94659bf7c8827077fc035, State: Initialized, Role: FOLLOWER
14:07:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.419042 18824 consensus_queue.cc:260] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } }
14:07:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.419301 18824 ts_tablet_manager.cc:1428] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
14:07:18.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.421088 18835 tablet_bootstrap.cc:654] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68: Neither blocks nor log segments found. Creating new log.
14:07:18.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.421993 18835 log.cc:826] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68: Log is configured to *not* fsync() on all Append() calls
14:07:18.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.423749 18835 tablet_bootstrap.cc:492] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68: No bootstrap required, opened a new log
14:07:18.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.423945 18835 ts_tablet_manager.cc:1397] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68: Time spent bootstrapping tablet: real 0.006s user 0.005s sys 0.000s
14:07:18.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.425339 18829 leader_election.cc:290] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241), a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229)
14:07:18.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.426122 18466 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9b182fc8b3834a57a90c369bd9f582cb" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20ee773143da465dbdafbd4e4afd6a7b" is_pre_election: true
14:07:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.426383 18466 raft_consensus.cc:2466] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 20377aa649e94659bf7c8827077fc035 in term 0.
14:07:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.426829 18683 leader_election.cc:304] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 20377aa649e94659bf7c8827077fc035, 20ee773143da465dbdafbd4e4afd6a7b; no voters:
14:07:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.427057 18835 raft_consensus.cc:357] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } }
14:07:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.427254 18835 raft_consensus.cc:383] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.427296 18835 raft_consensus.cc:738] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8cc3e270a1c4a4e96cb367ff07c7c68, State: Initialized, Role: FOLLOWER
14:07:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.427706 18835 consensus_queue.cc:260] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } }
14:07:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.428442 18663 heartbeater.cc:499] Master 127.17.164.126:34533 was elected leader, sending a full tablet report...
14:07:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.429348 18829 raft_consensus.cc:2802] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:07:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.429455 18829 raft_consensus.cc:491] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:07:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.429514 18829 raft_consensus.cc:3058] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Advancing to term 1
14:07:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.430202 18609 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9b182fc8b3834a57a90c369bd9f582cb" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" is_pre_election: true
14:07:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.430830 18829 raft_consensus.cc:513] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.430898 18835 ts_tablet_manager.cc:1428] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68: Time spent starting tablet: real 0.007s user 0.000s sys 0.004s
14:07:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.431185 18835 tablet_bootstrap.cc:492] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68: Bootstrap starting.
14:07:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.431139 18685 leader_election.cc:343] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229): Illegal state: must be running to vote when last-logged opid is not known
14:07:18.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.431617 18466 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9b182fc8b3834a57a90c369bd9f582cb" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20ee773143da465dbdafbd4e4afd6a7b"
14:07:18.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.431741 18466 raft_consensus.cc:3058] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Advancing to term 1
14:07:18.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.432859 18466 raft_consensus.cc:2466] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 20377aa649e94659bf7c8827077fc035 in term 1.
14:07:18.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.433192 18683 leader_election.cc:304] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 20377aa649e94659bf7c8827077fc035, 20ee773143da465dbdafbd4e4afd6a7b; no voters:
14:07:18.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.433315 18835 tablet_bootstrap.cc:654] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68: Neither blocks nor log segments found. Creating new log.
14:07:18.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.433382 18830 raft_consensus.cc:2802] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [term 1 FOLLOWER]: Leader election won for term 1
14:07:18.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.433722 18830 raft_consensus.cc:695] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [term 1 LEADER]: Becoming Leader. State: Replica: 20377aa649e94659bf7c8827077fc035, State: Running, Role: LEADER
14:07:18.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.434165 18830 consensus_queue.cc:237] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.434859 18609 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9b182fc8b3834a57a90c369bd9f582cb" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68"
14:07:18.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.435385 18685 leader_election.cc:343] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229): Illegal state: must be running to vote when last-logged opid is not known
14:07:18.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.435560 18829 leader_election.cc:290] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 election: Requested vote from peers 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241), a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229)
14:07:18.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.435636 18835 tablet_bootstrap.cc:492] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68: No bootstrap required, opened a new log
14:07:18.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.435708 18835 ts_tablet_manager.cc:1397] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
14:07:18.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.436087 18835 raft_consensus.cc:357] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.436235 18835 raft_consensus.cc:383] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:18.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.436272 18835 raft_consensus.cc:738] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8cc3e270a1c4a4e96cb367ff07c7c68, State: Initialized, Role: FOLLOWER
14:07:18.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.436391 18835 consensus_queue.cc:260] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.436683 18835 ts_tablet_manager.cc:1428] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
14:07:18.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.439188 18106 catalog_manager.cc:5582] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 reported cstate change: term changed from 0 to 1, leader changed from <none> to 20377aa649e94659bf7c8827077fc035 (127.17.164.67). New cstate: current_term: 1 leader_uuid: "20377aa649e94659bf7c8827077fc035" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } health_report { overall_health: HEALTHY } } }
14:07:18.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.453382 18337 catalog_manager.cc:1261] Loaded cluster ID: 1900d7fbdc174b3c9fae2ae657ce432f
14:07:18.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.453460 18337 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: loading cluster ID for follower catalog manager: success
14:07:18.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.455183 18337 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: acquiring CA information for follower catalog manager: success
14:07:18.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.456406 18337 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.463524 18350 catalog_manager.cc:1261] Loaded cluster ID: 1900d7fbdc174b3c9fae2ae657ce432f
14:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.463590 18350 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: loading cluster ID for follower catalog manager: success
14:07:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.465291 18350 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: acquiring CA information for follower catalog manager: success
14:07:18.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.466495 18350 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:07:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.479995 18837 raft_consensus.cc:491] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:07:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.480242 18837 raft_consensus.cc:513] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.481381 18837 leader_election.cc:290] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241), 20377aa649e94659bf7c8827077fc035 (127.17.164.67:37573)
14:07:18.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.485772 18466 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9b182fc8b3834a57a90c369bd9f582cb" candidate_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20ee773143da465dbdafbd4e4afd6a7b" is_pre_election: true
14:07:18.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.485864 18751 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "9b182fc8b3834a57a90c369bd9f582cb" candidate_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20377aa649e94659bf7c8827077fc035" is_pre_election: true
14:07:18.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.486012 18466 raft_consensus.cc:2391] T 9b182fc8b3834a57a90c369bd9f582cb P 20ee773143da465dbdafbd4e4afd6a7b [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a8cc3e270a1c4a4e96cb367ff07c7c68 in current term 1: Already voted for candidate 20377aa649e94659bf7c8827077fc035 in this term.
14:07:18.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.486508 18541 leader_election.cc:304] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a8cc3e270a1c4a4e96cb367ff07c7c68; no voters: 20377aa649e94659bf7c8827077fc035, 20ee773143da465dbdafbd4e4afd6a7b
14:07:18.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.486747 18837 raft_consensus.cc:3058] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Advancing to term 1
14:07:18.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.487701 18837 raft_consensus.cc:2747] T 9b182fc8b3834a57a90c369bd9f582cb P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:07:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.562532 18807 tablet.cc:2378] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:07:18.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.588572 18665 tablet.cc:2378] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:07:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.630688 18522 tablet.cc:2378] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:07:18.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.706660 18830 raft_consensus.cc:491] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:07:18.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.706825 18830 raft_consensus.cc:513] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } }
14:07:18.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.707227 18830 leader_election.cc:290] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229), 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241)
14:07:18.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.707775 18466 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "991f2c43aa654bb9a26fb5f15304e043" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20ee773143da465dbdafbd4e4afd6a7b" is_pre_election: true
14:07:18.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.707806 18609 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "991f2c43aa654bb9a26fb5f15304e043" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" is_pre_election: true
14:07:18.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.707901 18466 raft_consensus.cc:2466] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 20377aa649e94659bf7c8827077fc035 in term 0.
14:07:18.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.708060 18609 raft_consensus.cc:2466] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 20377aa649e94659bf7c8827077fc035 in term 0.
14:07:18.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.708261 18683 leader_election.cc:304] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 20377aa649e94659bf7c8827077fc035, 20ee773143da465dbdafbd4e4afd6a7b; no voters:
14:07:18.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.708456 18830 raft_consensus.cc:2802] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:07:18.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.708519 18830 raft_consensus.cc:491] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:07:18.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.708575 18830 raft_consensus.cc:3058] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Advancing to term 1
14:07:18.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.709446 18830 raft_consensus.cc:513] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } }
14:07:18.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.709828 18830 leader_election.cc:290] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 election: Requested vote from peers a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229), 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241)
14:07:18.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.710214 18466 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "991f2c43aa654bb9a26fb5f15304e043" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20ee773143da465dbdafbd4e4afd6a7b"
14:07:18.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.710237 18609 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "991f2c43aa654bb9a26fb5f15304e043" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68"
14:07:18.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.710320 18466 raft_consensus.cc:3058] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Advancing to term 1
14:07:18.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.710343 18609 raft_consensus.cc:3058] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Advancing to term 1
14:07:18.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.711068 18466 raft_consensus.cc:2466] T 991f2c43aa654bb9a26fb5f15304e043 P 20ee773143da465dbdafbd4e4afd6a7b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 20377aa649e94659bf7c8827077fc035 in term 1.
14:07:18.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.711163 18609 raft_consensus.cc:2466] T 991f2c43aa654bb9a26fb5f15304e043 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 20377aa649e94659bf7c8827077fc035 in term 1.
14:07:18.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.711355 18683 leader_election.cc:304] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 20377aa649e94659bf7c8827077fc035, 20ee773143da465dbdafbd4e4afd6a7b; no voters:
14:07:18.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.711544 18830 raft_consensus.cc:2802] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [term 1 FOLLOWER]: Leader election won for term 1
14:07:18.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.711642 18830 raft_consensus.cc:695] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [term 1 LEADER]: Becoming Leader. State: Replica: 20377aa649e94659bf7c8827077fc035, State: Running, Role: LEADER
14:07:18.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.711783 18830 consensus_queue.cc:237] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } }
14:07:18.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.713701 18107 catalog_manager.cc:5582] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 reported cstate change: term changed from 0 to 1, leader changed from <none> to 20377aa649e94659bf7c8827077fc035 (127.17.164.67). New cstate: current_term: 1 leader_uuid: "20377aa649e94659bf7c8827077fc035" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } health_report { overall_health: UNKNOWN } } }
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.765781 18107 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:43338:
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:07:18.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.766686 18107 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:07:18.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.775398 18588 tablet_service.cc:1468] Processing CreateTablet for tablet 5df4ed739ba24610b20e8d8bfc9616d8 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=219e1776ae3e40c18439e295cda92e80]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:07:18.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.776029 18588 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5df4ed739ba24610b20e8d8bfc9616d8. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.778254 18835 tablet_bootstrap.cc:492] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68: Bootstrap starting.
14:07:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.778601 18446 tablet_service.cc:1468] Processing CreateTablet for tablet 5df4ed739ba24610b20e8d8bfc9616d8 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=219e1776ae3e40c18439e295cda92e80]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:07:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.779286 18446 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5df4ed739ba24610b20e8d8bfc9616d8. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.779369 18835 tablet_bootstrap.cc:654] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68: Neither blocks nor log segments found. Creating new log.
14:07:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.779332 18731 tablet_service.cc:1468] Processing CreateTablet for tablet 5df4ed739ba24610b20e8d8bfc9616d8 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=219e1776ae3e40c18439e295cda92e80]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:07:18.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.779958 18731 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5df4ed739ba24610b20e8d8bfc9616d8. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:18.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.780927 18835 tablet_bootstrap.cc:492] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68: No bootstrap required, opened a new log
14:07:18.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.781061 18835 ts_tablet_manager.cc:1397] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
14:07:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.781668 18825 tablet_bootstrap.cc:492] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b: Bootstrap starting.
14:07:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.781690 18835 raft_consensus.cc:357] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.781801 18835 raft_consensus.cc:383] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.781834 18835 raft_consensus.cc:738] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8cc3e270a1c4a4e96cb367ff07c7c68, State: Initialized, Role: FOLLOWER
14:07:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.781971 18835 consensus_queue.cc:260] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.782275 18837 tablet_replica.cc:440] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } }
14:07:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.782413 18837 tablet_replica.cc:443] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68: This TxnStatusTablet replica's current role is: FOLLOWER
14:07:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.782719 18825 tablet_bootstrap.cc:654] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b: Neither blocks nor log segments found. Creating new log.
14:07:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.783685 18824 tablet_bootstrap.cc:492] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: Bootstrap starting.
14:07:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.784396 18825 tablet_bootstrap.cc:492] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b: No bootstrap required, opened a new log
14:07:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.784480 18825 ts_tablet_manager.cc:1397] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
14:07:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.784837 18824 tablet_bootstrap.cc:654] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: Neither blocks nor log segments found. Creating new log.
14:07:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.785022 18825 raft_consensus.cc:357] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.785100 18825 raft_consensus.cc:383] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.785121 18825 raft_consensus.cc:738] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 20ee773143da465dbdafbd4e4afd6a7b, State: Initialized, Role: FOLLOWER
14:07:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.785264 18825 consensus_queue.cc:260] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.785539 18825 ts_tablet_manager.cc:1428] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
14:07:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.785507 18834 tablet_replica.cc:440] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } }
14:07:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.785609 18834 tablet_replica.cc:443] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b: This TxnStatusTablet replica's current role is: FOLLOWER
14:07:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.782274 18835 ts_tablet_manager.cc:1428] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
14:07:18.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.788365 18824 tablet_bootstrap.cc:492] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: No bootstrap required, opened a new log
14:07:18.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.788476 18824 ts_tablet_manager.cc:1397] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
14:07:18.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.789193 18824 raft_consensus.cc:357] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.789316 18824 raft_consensus.cc:383] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:18.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.789355 18824 raft_consensus.cc:738] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 20377aa649e94659bf7c8827077fc035, State: Initialized, Role: FOLLOWER
14:07:18.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.789489 18824 consensus_queue.cc:260] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.789731 18824 ts_tablet_manager.cc:1428] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
14:07:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.789765 18830 tablet_replica.cc:440] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } }
14:07:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.789899 18830 tablet_replica.cc:443] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: This TxnStatusTablet replica's current role is: FOLLOWER
14:07:18.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.793124 18830 raft_consensus.cc:491] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:07:18.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.793215 18830 raft_consensus.cc:513] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.793589 18830 leader_election.cc:290] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241), a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229)
14:07:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.793992 18609 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5df4ed739ba24610b20e8d8bfc9616d8" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" is_pre_election: true
14:07:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.794133 18609 raft_consensus.cc:2466] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 20377aa649e94659bf7c8827077fc035 in term 0.
14:07:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.794209 18466 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5df4ed739ba24610b20e8d8bfc9616d8" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20ee773143da465dbdafbd4e4afd6a7b" is_pre_election: true
14:07:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.794368 18466 raft_consensus.cc:2466] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 20377aa649e94659bf7c8827077fc035 in term 0.
14:07:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.794409 18685 leader_election.cc:304] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 20377aa649e94659bf7c8827077fc035, a8cc3e270a1c4a4e96cb367ff07c7c68; no voters:
14:07:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.794600 18830 raft_consensus.cc:2802] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:07:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.794641 18830 raft_consensus.cc:491] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:07:18.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.794667 18830 raft_consensus.cc:3058] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [term 0 FOLLOWER]: Advancing to term 1
14:07:18.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.795442 18830 raft_consensus.cc:513] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.795740 18830 leader_election.cc:290] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 election: Requested vote from peers 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241), a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229)
14:07:18.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.796063 18466 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5df4ed739ba24610b20e8d8bfc9616d8" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20ee773143da465dbdafbd4e4afd6a7b"
14:07:18.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.796202 18466 raft_consensus.cc:3058] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b [term 0 FOLLOWER]: Advancing to term 1
14:07:18.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.796140 18609 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "5df4ed739ba24610b20e8d8bfc9616d8" candidate_uuid: "20377aa649e94659bf7c8827077fc035" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68"
14:07:18.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.796290 18609 raft_consensus.cc:3058] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 0 FOLLOWER]: Advancing to term 1
14:07:18.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.796989 18466 raft_consensus.cc:2466] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 20377aa649e94659bf7c8827077fc035 in term 1.
14:07:18.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.797051 18609 raft_consensus.cc:2466] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 20377aa649e94659bf7c8827077fc035 in term 1.
14:07:18.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.797288 18683 leader_election.cc:304] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 20377aa649e94659bf7c8827077fc035, 20ee773143da465dbdafbd4e4afd6a7b; no voters:
14:07:18.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.797454 18830 raft_consensus.cc:2802] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [term 1 FOLLOWER]: Leader election won for term 1
14:07:18.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.797562 18830 raft_consensus.cc:695] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [term 1 LEADER]: Becoming Leader. State: Replica: 20377aa649e94659bf7c8827077fc035, State: Running, Role: LEADER
14:07:18.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.797698 18830 consensus_queue.cc:237] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } }
14:07:18.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.798491 18829 tablet_replica.cc:440] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: TxnStatusTablet state changed. Reason: New leader 20377aa649e94659bf7c8827077fc035. Latest consensus state: current_term: 1 leader_uuid: "20377aa649e94659bf7c8827077fc035" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } }
14:07:18.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.798611 18829 tablet_replica.cc:443] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: This TxnStatusTablet replica's current role is: LEADER
14:07:18.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.799487 18106 catalog_manager.cc:5582] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 reported cstate change: term changed from 0 to 1, leader changed from <none> to 20377aa649e94659bf7c8827077fc035 (127.17.164.67). New cstate: current_term: 1 leader_uuid: "20377aa649e94659bf7c8827077fc035" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } health_report { overall_health: HEALTHY } } }
14:07:18.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.800310 18855 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
14:07:18.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.800498 18855 txn_status_manager.cc:930] Loading transaction status metadata into memory...
14:07:18.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.817978 18466 raft_consensus.cc:1273] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b [term 1 FOLLOWER]: Refusing update from remote peer 20377aa649e94659bf7c8827077fc035: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:07:18.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.818137 18609 raft_consensus.cc:1273] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68 [term 1 FOLLOWER]: Refusing update from remote peer 20377aa649e94659bf7c8827077fc035: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:07:18.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.818655 18830 consensus_queue.cc:1035] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [LEADER]: Connected to new peer: Peer: permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:18.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.818811 18829 consensus_queue.cc:1035] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.822306 18837 tablet_replica.cc:440] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68: TxnStatusTablet state changed. Reason: New leader 20377aa649e94659bf7c8827077fc035. Latest consensus state: current_term: 1 leader_uuid: "20377aa649e94659bf7c8827077fc035" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } }
14:07:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.822425 18837 tablet_replica.cc:443] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68: This TxnStatusTablet replica's current role is: FOLLOWER
14:07:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.822728 18834 tablet_replica.cc:440] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b: TxnStatusTablet state changed. Reason: New leader 20377aa649e94659bf7c8827077fc035. Latest consensus state: current_term: 1 leader_uuid: "20377aa649e94659bf7c8827077fc035" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } }
14:07:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.822830 18834 tablet_replica.cc:443] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b: This TxnStatusTablet replica's current role is: FOLLOWER
14:07:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.824342 18830 tablet_replica.cc:440] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "20377aa649e94659bf7c8827077fc035" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } }
14:07:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.824433 18830 tablet_replica.cc:443] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: This TxnStatusTablet replica's current role is: LEADER
14:07:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.824530 18858 mvcc.cc:204] Tried to move back new op lower bound from 7170462674192228352 to 7170462674117144576. Current Snapshot: MvccSnapshot[applied={T|T < 7170462674192228352}]
14:07:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.826344 18837 tablet_replica.cc:440] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "20377aa649e94659bf7c8827077fc035" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } }
14:07:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.826437 18837 tablet_replica.cc:443] T 5df4ed739ba24610b20e8d8bfc9616d8 P a8cc3e270a1c4a4e96cb367ff07c7c68: This TxnStatusTablet replica's current role is: FOLLOWER
14:07:18.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.830175 18859 mvcc.cc:204] Tried to move back new op lower bound from 7170462674192228352 to 7170462674117144576. Current Snapshot: MvccSnapshot[applied={T|T < 7170462674192228352}]
14:07:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.831281 18829 tablet_replica.cc:440] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "20377aa649e94659bf7c8827077fc035" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } }
14:07:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.831380 18829 tablet_replica.cc:443] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20377aa649e94659bf7c8827077fc035: This TxnStatusTablet replica's current role is: LEADER
14:07:18.834 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
14:07:18.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.835304 18834 tablet_replica.cc:440] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "20377aa649e94659bf7c8827077fc035" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 } } peers { permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 } } peers { permanent_uuid: "20377aa649e94659bf7c8827077fc035" member_type: VOTER last_known_addr { host: "127.17.164.67" port: 37573 } } }
14:07:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.835423 18834 tablet_replica.cc:443] T 5df4ed739ba24610b20e8d8bfc9616d8 P 20ee773143da465dbdafbd4e4afd6a7b: This TxnStatusTablet replica's current role is: FOLLOWER
14:07:18.836 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 93e6eaae358242959886acaf4bb50364
14:07:18.837 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.164.126:34533
14:07:18.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:07:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.837875 18860 mvcc.cc:204] Tried to move back new op lower bound from 7170462674192228352 to 7170462674117144576. Current Snapshot: MvccSnapshot[applied={T|T < 7170462674192228352}]
14:07:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.838124 18065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu with pid 18076
14:07:18.846 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.17.164.126:34533(127.17.164.126:34533)] handling channelInactive
14:07:18.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state READY due to: connection closed
14:07:18.847 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:18.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.890170 18838 consensus_queue.cc:1035] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:18.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.892597 18663 heartbeater.cc:646] Failed to heartbeat to 127.17.164.126:34533 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111)
14:07:18.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.896427 18805 heartbeater.cc:646] Failed to heartbeat to 127.17.164.126:34533 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111)
14:07:18.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:18.896587 18829 consensus_queue.cc:1035] T 9b182fc8b3834a57a90c369bd9f582cb P 20377aa649e94659bf7c8827077fc035 [LEADER]: Connected to new peer: Peer: permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:18.898918 18520 heartbeater.cc:646] Failed to heartbeat to 127.17.164.126:34533 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111)
14:07:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.104614 18870 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 68e4a681ed9d4cc6bb513c955989849f)
14:07:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.104739 18870 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.105311 18871 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 68e4a681ed9d4cc6bb513c955989849f)
14:07:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.105422 18871 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.105640 18870 leader_election.cc:290] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689)
14:07:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:19.106233 18255 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111)
14:07:19.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.106369 18199 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8e81f7c39b084eb1863771945700e98e" candidate_term: 3 candidate_status { last_received { term: 2 index: 11 } } ignore_live_leader: false dest_uuid: "3c2b262c9aae4117ae997f8710cd51ae" is_pre_election: true
14:07:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.106509 18199 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8e81f7c39b084eb1863771945700e98e in term 2.
14:07:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.106809 18280 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c2b262c9aae4117ae997f8710cd51ae" candidate_term: 3 candidate_status { last_received { term: 2 index: 11 } } ignore_live_leader: false dest_uuid: "8e81f7c39b084eb1863771945700e98e" is_pre_election: true
14:07:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.106504 18871 leader_election.cc:290] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213)
14:07:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.106922 18280 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3c2b262c9aae4117ae997f8710cd51ae in term 2.
14:07:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:19.106940 18173 leader_election.cc:336] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533): Network error: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111)
14:07:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.106990 18257 leader_election.cc:304] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3c2b262c9aae4117ae997f8710cd51ae, 8e81f7c39b084eb1863771945700e98e; no voters:
14:07:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.107213 18870 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 2 FOLLOWER]: Leader pre-election won for term 3
14:07:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.107254 18870 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 2 FOLLOWER]: Starting leader election (detected failure of leader 68e4a681ed9d4cc6bb513c955989849f)
14:07:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.107280 18870 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 2 FOLLOWER]: Advancing to term 3
14:07:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.107802 18175 leader_election.cc:304] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3c2b262c9aae4117ae997f8710cd51ae, 8e81f7c39b084eb1863771945700e98e; no voters: 68e4a681ed9d4cc6bb513c955989849f
14:07:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:19.107843 18255 leader_election.cc:336] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533): Network error: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111)
14:07:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.108042 18871 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 2 FOLLOWER]: Leader pre-election won for term 3
14:07:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.108139 18871 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 2 FOLLOWER]: Starting leader election (detected failure of leader 68e4a681ed9d4cc6bb513c955989849f)
14:07:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.108197 18871 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 2 FOLLOWER]: Advancing to term 3
14:07:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.108326 18870 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.108667 18870 leader_election.cc:290] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 3 election: Requested vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689)
14:07:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.109031 18199 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8e81f7c39b084eb1863771945700e98e" candidate_term: 3 candidate_status { last_received { term: 2 index: 11 } } ignore_live_leader: false dest_uuid: "3c2b262c9aae4117ae997f8710cd51ae"
14:07:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.109237 18871 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.109457 18199 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 3 FOLLOWER]: Leader election vote request: Denying vote to candidate 8e81f7c39b084eb1863771945700e98e in current term 3: Already voted for candidate 3c2b262c9aae4117ae997f8710cd51ae in this term.
14:07:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:19.109509 18255 leader_election.cc:336] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533): Network error: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111)
14:07:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.109730 18871 leader_election.cc:290] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 3 election: Requested vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213)
14:07:19.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.109874 18280 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c2b262c9aae4117ae997f8710cd51ae" candidate_term: 3 candidate_status { last_received { term: 2 index: 11 } } ignore_live_leader: false dest_uuid: "8e81f7c39b084eb1863771945700e98e"
14:07:19.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.110016 18280 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 3 FOLLOWER]: Leader election vote request: Denying vote to candidate 3c2b262c9aae4117ae997f8710cd51ae in current term 3: Already voted for candidate 8e81f7c39b084eb1863771945700e98e in this term.
14:07:19.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.109969 18257 leader_election.cc:304] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 3 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8e81f7c39b084eb1863771945700e98e; no voters: 3c2b262c9aae4117ae997f8710cd51ae, 68e4a681ed9d4cc6bb513c955989849f
14:07:19.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.110158 18870 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 3 FOLLOWER]: Leader election lost for term 3. Reason: could not achieve majority
14:07:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:19.111004 18173 leader_election.cc:336] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533): Network error: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111)
14:07:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.111063 18173 leader_election.cc:304] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 3 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3c2b262c9aae4117ae997f8710cd51ae; no voters: 68e4a681ed9d4cc6bb513c955989849f, 8e81f7c39b084eb1863771945700e98e
14:07:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.111182 18871 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 3 FOLLOWER]: Leader election lost for term 3. Reason: could not achieve majority
14:07:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.145582 18838 consensus_queue.cc:1035] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [LEADER]: Connected to new peer: Peer: permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" member_type: VOTER last_known_addr { host: "127.17.164.65" port: 37241 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.149103 18830 consensus_queue.cc:1035] T 991f2c43aa654bb9a26fb5f15304e043 P 20377aa649e94659bf7c8827077fc035 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" member_type: VOTER last_known_addr { host: "127.17.164.66" port: 45229 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:19.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:19.351 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.351 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.351 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.17.164.126:34533(127.17.164.126:34533) for tablet Kudu Master: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.369 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:19.372 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.126:34533: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:19.389 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:19.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.391 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.126:34533: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.392 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:19.410 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:19.413 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.126:34533: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.413 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:19.429 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:19.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.431 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.126:34533: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:19.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:19.451 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.451 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.126:34533: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.453 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:19.469 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:19.470 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.471 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.126:34533: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:19.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:19.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.511 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.126:34533: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:19.513 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] sending RPC to server master-127.17.164.126:34533, [1ms] received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [21ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [22ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [63ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [102ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [103ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [120ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [121ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [161ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [162ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, deferred=Deferred@1665830626(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@27e262f3 -> (continuation of Deferred@1876127417 after retry RPC after error@880818286) -> (continuation of Deferred@59411842 after retry RPC after error@936088314) -> (continuation of Deferred@1134138396 after retry RPC after error@1987020617) -> (continuation of Deferred@559870619 after retry RPC after error@1248458760) -> (continuation of Deferred@1163897901 after retry RPC after error@1325411840) -> (continuation of Deferred@429709137 after retry RPC after error@481351122), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@24b62cad -> (continuation of Deferred@1876127417 after retry RPC after error@880818286) -> (continuation of Deferred@59411842 after retry RPC after error@936088314) -> (continuation of Deferred@1134138396 after retry RPC after error@1987020617) -> (continuation of Deferred@559870619 after retry RPC after error@1248458760) -> (continuation of Deferred@1163897901 after retry RPC after error@1325411840) -> (continuation of Deferred@429709137 after retry RPC after error@481351122)))
14:07:19.514 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] sending RPC to server master-127.17.164.126:34533, [1ms] received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [21ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [22ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [63ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [102ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [103ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [120ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [121ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [161ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [162ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, deferred=Deferred@1665830626(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@27e262f3 -> (continuation of Deferred@1876127417 after retry RPC after error@880818286) -> (continuation of Deferred@59411842 after retry RPC after error@936088314) -> (continuation of Deferred@1134138396 after retry RPC after error@1987020617) -> (continuation of Deferred@559870619 after retry RPC after error@1248458760) -> (continuation of Deferred@1163897901 after retry RPC after error@1325411840) -> (continuation of Deferred@429709137 after retry RPC after error@481351122), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@24b62cad -> (continuation of Deferred@1876127417 after retry RPC after error@880818286) -> (continuation of Deferred@59411842 after retry RPC after error@936088314) -> (continuation of Deferred@1134138396 after retry RPC after error@1987020617) -> (continuation of Deferred@559870619 after retry RPC after error@1248458760) -> (continuation of Deferred@1163897901 after retry RPC after error@1325411840) -> (continuation of Deferred@429709137 after retry RPC after error@481351122)))
14:07:19.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:19.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.126:34533: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.532 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:19.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.538178 18870 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:07:19.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.538379 18870 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:19.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.538882 18870 leader_election.cc:290] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689)
14:07:19.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.539515 18199 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8e81f7c39b084eb1863771945700e98e" candidate_term: 4 candidate_status { last_received { term: 2 index: 11 } } ignore_live_leader: false dest_uuid: "3c2b262c9aae4117ae997f8710cd51ae" is_pre_election: true
14:07:19.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.539713 18199 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8e81f7c39b084eb1863771945700e98e in term 3.
14:07:19.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.540079 18257 leader_election.cc:304] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3c2b262c9aae4117ae997f8710cd51ae, 8e81f7c39b084eb1863771945700e98e; no voters:
14:07:19.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:19.540223 18255 leader_election.cc:336] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533): Network error: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111)
14:07:19.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.540323 18870 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 3 FOLLOWER]: Leader pre-election won for term 4
14:07:19.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.540370 18870 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:07:19.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.540412 18870 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 3 FOLLOWER]: Advancing to term 4
14:07:19.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.541404 18870 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:19.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.541754 18870 leader_election.cc:290] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 4 election: Requested vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689)
14:07:19.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.542210 18199 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8e81f7c39b084eb1863771945700e98e" candidate_term: 4 candidate_status { last_received { term: 2 index: 11 } } ignore_live_leader: false dest_uuid: "3c2b262c9aae4117ae997f8710cd51ae"
14:07:19.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.542347 18199 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 3 FOLLOWER]: Advancing to term 4
14:07:19.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:19.542702 18255 leader_election.cc:336] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533): Network error: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111)
14:07:19.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.543442 18199 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8e81f7c39b084eb1863771945700e98e in term 4.
14:07:19.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.543751 18257 leader_election.cc:304] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3c2b262c9aae4117ae997f8710cd51ae, 8e81f7c39b084eb1863771945700e98e; no voters: 68e4a681ed9d4cc6bb513c955989849f
14:07:19.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.543924 18870 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 4 FOLLOWER]: Leader election won for term 4
14:07:19.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.544216 18870 raft_consensus.cc:695] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 4 LEADER]: Becoming Leader. State: Replica: 8e81f7c39b084eb1863771945700e98e, State: Running, Role: LEADER
14:07:19.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.544613 18870 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 2.11, Last appended by leader: 11, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.545668 18879 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8e81f7c39b084eb1863771945700e98e. Latest consensus state: current_term: 4 leader_uuid: "8e81f7c39b084eb1863771945700e98e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.545755 18879 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: This master's current role is: LEADER
14:07:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.546039 18880 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:07:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.547735 18880 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=219e1776ae3e40c18439e295cda92e80]
14:07:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.548166 18880 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364]
14:07:19.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.549309 18880 tablet_loader.cc:96] loaded metadata for tablet 5df4ed739ba24610b20e8d8bfc9616d8 (table kudu_system.kudu_transactions [id=219e1776ae3e40c18439e295cda92e80])
14:07:19.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.549624 18880 tablet_loader.cc:96] loaded metadata for tablet 991f2c43aa654bb9a26fb5f15304e043 (table txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364])
14:07:19.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.549813 18880 tablet_loader.cc:96] loaded metadata for tablet 9b182fc8b3834a57a90c369bd9f582cb (table txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364])
14:07:19.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.549961 18880 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:07:19.550 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:19.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.550474 18880 catalog_manager.cc:1261] Loaded cluster ID: 1900d7fbdc174b3c9fae2ae657ce432f
14:07:19.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.550531 18880 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:07:19.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.551252 18880 catalog_manager.cc:1506] Loading token signing keys...
14:07:19.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.551708 18880 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: Loaded TSK: 0
14:07:19.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.126:34533: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.553122 18880 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:07:19.553 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:19.570 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:19.571 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.571 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.126:34533: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533
14:07:19.573 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:07:19.574 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=3, TimeoutTracker(timeout=250, elapsed=44), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [2ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [3ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [21ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [22ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [23ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: Network error: Failed to connect to peer master-127.17.164.126:34533(127.17.164.126:34533): Connection refused: /127.17.164.126:34533, [42ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [43ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, deferred=Deferred@864205037(state=PENDING, result=null, callback=(continuation of Deferred@777830149 after retry RPC after error@200792010) -> (continuation of Deferred@252400557 after retry RPC after error@694736753), errback=(continuation of Deferred@777830149 after retry RPC after error@200792010) -> (continuation of Deferred@252400557 after retry RPC after error@694736753))) after lookup
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.607230 18267 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36880:
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:07:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:19.608741 18267 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:07:19.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.635110 18199 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 4 FOLLOWER]: Refusing update from remote peer 8e81f7c39b084eb1863771945700e98e: Log matching property violated. Preceding OpId in replica: term: 2 index: 11. Preceding OpId from leader: term: 4 index: 12. (index mismatch)
14:07:19.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.635761 18879 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [LEADER]: Connected to new peer: Peer: permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
14:07:19.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.637085 18893 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8e81f7c39b084eb1863771945700e98e. Latest consensus state: current_term: 4 leader_uuid: "8e81f7c39b084eb1863771945700e98e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:19.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.637202 18893 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: This master's current role is: FOLLOWER
14:07:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.638324 18870 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "8e81f7c39b084eb1863771945700e98e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.638453 18870 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: This master's current role is: LEADER
14:07:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.638607 18893 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "8e81f7c39b084eb1863771945700e98e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:19.638823 18893 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: This master's current role is: FOLLOWER
14:07:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:19.639520 18255 consensus_peers.cc:487] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e -> Peer 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533): Couldn't send request to peer 68e4a681ed9d4cc6bb513c955989849f. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:07:20.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:20.106277 18255 consensus_peers.cc:487] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e -> Peer 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533): Couldn't send request to peer 68e4a681ed9d4cc6bb513c955989849f. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:07:20.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:20.575315 18255 consensus_peers.cc:487] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e -> Peer 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533): Couldn't send request to peer 68e4a681ed9d4cc6bb513c955989849f. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
14:07:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:20.584544 18804 heartbeater.cc:499] Master 127.17.164.124:33213 was elected leader, sending a full tablet report...
14:07:20.854 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 991f2c43aa654bb9a26fb5f15304e043 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
14:07:20.854 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 9b182fc8b3834a57a90c369bd9f582cb for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
14:07:20.854 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=991f2c43aa654bb9a26fb5f15304e043@[20377aa649e94659bf7c8827077fc035(127.17.164.67:37573)[L],20ee773143da465dbdafbd4e4afd6a7b(127.17.164.65:37241),a8cc3e270a1c4a4e96cb367ff07c7c68(127.17.164.66:45229)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=9b182fc8b3834a57a90c369bd9f582cb@[20377aa649e94659bf7c8827077fc035(127.17.164.67:37573)[L],20ee773143da465dbdafbd4e4afd6a7b(127.17.164.65:37241),a8cc3e270a1c4a4e96cb367ff07c7c68(127.17.164.66:45229)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
14:07:20.854 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.17.164.124:33213, [2ms] Sub RPC GetTableLocations: received response from server master-127.17.164.124:33213: OK, deferred=null) after lookup
14:07:20.855 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 20377aa649e94659bf7c8827077fc035(127.17.164.67:37573)] connecting to peer
14:07:20.856 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 20377aa649e94659bf7c8827077fc035(127.17.164.67:37573)] Successfully connected to peer
14:07:20.871 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9254cf09, L:/127.0.0.1:54768 - R:/127.17.164.67:37573] using TOKEN/null
14:07:20.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:20.875525 18772 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.17.164.126:34533)
14:07:20.903 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.164.126:34533
14:07:20.904 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:20.904819 18065 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/logs
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data/info.pb
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.126:34533
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.126
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34697
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33855/jwks.json
14:07:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:07:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:07:21.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.037927 18918 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:21.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.038295 18918 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:21.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.038384 18918 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.042426 18918 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.042517 18918 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:07:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.042547 18918 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.042574 18918 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.042599 18918 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.042626 18918 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.042645 18918 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33855/jwks.json
14:07:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.047529 18918 master_runner.cc:386] Master server non-default flags:
14:07:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.126:34533
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33855/jwks.json
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data/info.pb
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.126
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34697
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.18918
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/logs
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:13 UTC on 24a791456cd2
14:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6714
14:07:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.049484 18918 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:21.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.051020 18918 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.060395 18923 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.060364 18926 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.060590 18918 server_base.cc:1048] running on GCE node
14:07:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.060422 18924 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:21.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.061769 18918 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:21.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.063093 18918 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:21.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.064224 18918 hybrid_clock.cc:648] HybridClock initialized: now 1750601241064197 us; error 53 us; skew 500 ppm
14:07:21.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.067340 18918 webserver.cc:469] Webserver started at http://127.17.164.126:34697/ using document root <none> and password file <none>
14:07:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.068282 18918 fs_manager.cc:362] Metadata directory not provided
14:07:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.068394 18918 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:21.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.073165 18918 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.003s sys 0.000s
14:07:21.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.076222 18932 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.077795 18918 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.077970 18918 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data,/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "68e4a681ed9d4cc6bb513c955989849f"
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:16 on dist-test-slave-77bb"
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.078583 18918 fs_report.cc:389] FS layout report
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/wal
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data/data
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.099120 18918 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.100159 18918 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.100382 18918 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.104595 18255 consensus_peers.cc:487] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e -> Peer 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533): Couldn't send request to peer 68e4a681ed9d4cc6bb513c955989849f. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.126:34533: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
14:07:21.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.111091 18918 rpc_server.cc:307] RPC server started. Bound to: 127.17.164.126:34533
14:07:21.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.111102 18994 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.164.126:34533 every 8 connection(s)
14:07:21.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.112653 18918 server_base.cc:1180] Dumped server information to /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-0/data/info.pb
14:07:21.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.113247 18065 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu as pid 18918
14:07:21.113 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:21.113 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.164.124:33213
14:07:21.114 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:07:21.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.114310 18065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu with pid 18237
14:07:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.118988 18995 sys_catalog.cc:263] Verifying existing consensus state
14:07:21.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.121133 18995 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: Bootstrap starting.
14:07:21.123 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.17.164.124:33213(127.17.164.124:33213)] handling channelInactive
14:07:21.123 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.124:33213(127.17.164.124:33213)] cleaning up while in state READY due to: connection closed
14:07:21.125 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:21.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.139140 18995 log.cc:826] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: Log is configured to *not* fsync() on all Append() calls
14:07:21.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.150033 18995 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:07:21.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.150467 18995 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: Bootstrap complete.
14:07:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.153137 18519 heartbeater.cc:499] Master 127.17.164.124:33213 was elected leader, sending a full tablet report...
14:07:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.153514 18662 heartbeater.cc:499] Master 127.17.164.124:33213 was elected leader, sending a full tablet report...
14:07:21.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.156718 18995 raft_consensus.cc:357] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:21.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.157161 18995 raft_consensus.cc:738] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 68e4a681ed9d4cc6bb513c955989849f, State: Initialized, Role: FOLLOWER
14:07:21.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.157802 18995 consensus_queue.cc:260] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 2.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:21.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.158473 19004 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:21.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.158746 19004 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: This master's current role is: FOLLOWER
14:07:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.159394 18995 sys_catalog.cc:564] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: configured and running, proceeding with master startup.
14:07:21.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.164131 18663 heartbeater.cc:344] Connected to a master server at 127.17.164.126:34533
14:07:21.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.164158 18520 heartbeater.cc:344] Connected to a master server at 127.17.164.126:34533
14:07:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.166206 18948 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" instance_seqno: 1750601237873680) as {username='slave'} at 127.17.164.65:49149; Asking this server to re-register.
14:07:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.166657 19015 catalog_manager.cc:1261] Loaded cluster ID: 1900d7fbdc174b3c9fae2ae657ce432f
14:07:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.166719 19015 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: loading cluster ID for follower catalog manager: success
14:07:21.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.167213 18520 heartbeater.cc:461] Registering TS with master...
14:07:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.168346 18948 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" instance_seqno: 1750601238080687) as {username='slave'} at 127.17.164.66:33781; Asking this server to re-register.
14:07:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.168979 18949 ts_manager.cc:194] Registered new tserver with Master: 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241)
14:07:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.169414 19015 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: acquiring CA information for follower catalog manager: success
14:07:21.169 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.124:33213(127.17.164.124:33213)] connecting to peer
14:07:21.171 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.124:33213(127.17.164.124:33213)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.171 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.17.164.124:33213(127.17.164.124:33213) for tablet Kudu Master: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.170795 18663 heartbeater.cc:461] Registering TS with master...
14:07:21.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.172164 18949 ts_manager.cc:194] Registered new tserver with Master: a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229)
14:07:21.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.172799 19015 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:07:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.179047 18918 master_runner.cc:186] Error getting master registration for 127.17.164.124:33213: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111),
14:07:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.180771 18918 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:21.189 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.124:33213(127.17.164.124:33213)] connecting to peer
14:07:21.190 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.126:34533(127.17.164.126:34533)] connecting to peer
14:07:21.191 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.17.164.126:34533(127.17.164.126:34533)] Successfully connected to peer
14:07:21.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.191 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.124:33213(127.17.164.124:33213)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.191 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.124:33213: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.205 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x8a99a9a8, L:/127.0.0.1:43824 - R:/127.17.164.126:34533] using TOKEN/null
14:07:21.207 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:21.229 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.124:33213(127.17.164.124:33213)] connecting to peer
14:07:21.231 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.231 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.124:33213(127.17.164.124:33213)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.231 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.124:33213: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.232 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:21.249 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.124:33213(127.17.164.124:33213)] connecting to peer
14:07:21.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.124:33213(127.17.164.124:33213)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.251 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.124:33213: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.252 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:21.270 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.124:33213(127.17.164.124:33213)] connecting to peer
14:07:21.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.271 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.124:33213(127.17.164.124:33213)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.124:33213: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:21.289 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.124:33213(127.17.164.124:33213)] connecting to peer
14:07:21.291 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.291 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.124:33213(127.17.164.124:33213)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.291 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.124:33213: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:21.309 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.124:33213(127.17.164.124:33213)] connecting to peer
14:07:21.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.124:33213(127.17.164.124:33213)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.313 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.124:33213: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.313 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:21.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.368095 19019 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 8e81f7c39b084eb1863771945700e98e)
14:07:21.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.368383 19019 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.369338 19019 leader_election.cc:290] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213)
14:07:21.369 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.124:33213(127.17.164.124:33213)] connecting to peer
14:07:21.370 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.371 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.124:33213(127.17.164.124:33213)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.371 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.124:33213: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.371744 18175 leader_election.cc:336] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213): Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111)
14:07:21.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:21.373 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] sending RPC to server master-127.17.164.124:33213, [2ms] received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [22ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [25ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [38ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [38ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [82ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [83ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [102ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [103ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [122ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [122ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [143ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [144ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [202ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [202ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, deferred=Deferred@665906838(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@5a5703e2 -> (continuation of Deferred@487247660 after retry RPC after error@2057699595) -> (continuation of Deferred@1514312855 after retry RPC after error@946390062) -> (continuation of Deferred@1934630816 after retry RPC after error@968156824) -> (continuation of Deferred@1998334741 after retry RPC after error@153868396) -> (continuation of Deferred@2054515702 after retry RPC after error@1684787718) -> (continuation of Deferred@1924733213 after retry RPC after error@621172638), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@747db213 -> (continuation of Deferred@487247660 after retry RPC after error@2057699595) -> (continuation of Deferred@1514312855 after retry RPC after error@946390062) -> (continuation of Deferred@1934630816 after retry RPC after error@968156824) -> (continuation of Deferred@1998334741 after retry RPC after error@153868396) -> (continuation of Deferred@2054515702 after retry RPC after error@1684787718) -> (continuation of Deferred@1924733213 after retry RPC after error@621172638)))
14:07:21.373 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] sending RPC to server master-127.17.164.124:33213, [2ms] received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [22ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [25ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [38ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [38ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [82ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [83ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [102ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [103ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [122ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [122ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [143ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [144ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [202ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [202ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, deferred=Deferred@665906838(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@5a5703e2 -> (continuation of Deferred@487247660 after retry RPC after error@2057699595) -> (continuation of Deferred@1514312855 after retry RPC after error@946390062) -> (continuation of Deferred@1934630816 after retry RPC after error@968156824) -> (continuation of Deferred@1998334741 after retry RPC after error@153868396) -> (continuation of Deferred@2054515702 after retry RPC after error@1684787718) -> (continuation of Deferred@1924733213 after retry RPC after error@621172638), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@747db213 -> (continuation of Deferred@487247660 after retry RPC after error@2057699595) -> (continuation of Deferred@1514312855 after retry RPC after error@946390062) -> (continuation of Deferred@1934630816 after retry RPC after error@968156824) -> (continuation of Deferred@1998334741 after retry RPC after error@153868396) -> (continuation of Deferred@2054515702 after retry RPC after error@1684787718) -> (continuation of Deferred@1924733213 after retry RPC after error@621172638)))
14:07:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.374459 18959 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c2b262c9aae4117ae997f8710cd51ae" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "68e4a681ed9d4cc6bb513c955989849f" is_pre_election: true
14:07:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.374941 18959 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3c2b262c9aae4117ae997f8710cd51ae in term 2.
14:07:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.375401 18173 leader_election.cc:304] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3c2b262c9aae4117ae997f8710cd51ae, 68e4a681ed9d4cc6bb513c955989849f; no voters: 8e81f7c39b084eb1863771945700e98e
14:07:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.375617 19019 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 4 FOLLOWER]: Leader pre-election won for term 5
14:07:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.375689 19019 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 4 FOLLOWER]: Starting leader election (detected failure of leader 8e81f7c39b084eb1863771945700e98e)
14:07:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.375730 19019 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 4 FOLLOWER]: Advancing to term 5
14:07:21.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.377110 19019 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:21.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.377564 19019 leader_election.cc:290] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 5 election: Requested vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213)
14:07:21.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.377969 18959 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c2b262c9aae4117ae997f8710cd51ae" candidate_term: 5 candidate_status { last_received { term: 4 index: 12 } } ignore_live_leader: false dest_uuid: "68e4a681ed9d4cc6bb513c955989849f"
14:07:21.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.378146 18959 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 2 FOLLOWER]: Advancing to term 5
14:07:21.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.378728 18175 leader_election.cc:336] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213): Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111)
14:07:21.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.380640 18959 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3c2b262c9aae4117ae997f8710cd51ae in term 5.
14:07:21.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.381078 18173 leader_election.cc:304] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3c2b262c9aae4117ae997f8710cd51ae, 68e4a681ed9d4cc6bb513c955989849f; no voters: 8e81f7c39b084eb1863771945700e98e
14:07:21.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.381325 19019 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 5 FOLLOWER]: Leader election won for term 5
14:07:21.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.381606 19019 raft_consensus.cc:695] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [term 5 LEADER]: Becoming Leader. State: Replica: 3c2b262c9aae4117ae997f8710cd51ae, State: Running, Role: LEADER
14:07:21.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.381795 19019 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:21.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.382591 19023 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3c2b262c9aae4117ae997f8710cd51ae. Latest consensus state: current_term: 5 leader_uuid: "3c2b262c9aae4117ae997f8710cd51ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:21.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.382687 19023 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: This master's current role is: LEADER
14:07:21.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.382978 19025 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:07:21.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.384764 19025 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=219e1776ae3e40c18439e295cda92e80]
14:07:21.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.385141 19025 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364]
14:07:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.386261 19025 tablet_loader.cc:96] loaded metadata for tablet 5df4ed739ba24610b20e8d8bfc9616d8 (table kudu_system.kudu_transactions [id=219e1776ae3e40c18439e295cda92e80])
14:07:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.386539 19025 tablet_loader.cc:96] loaded metadata for tablet 991f2c43aa654bb9a26fb5f15304e043 (table txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364])
14:07:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.386716 19025 tablet_loader.cc:96] loaded metadata for tablet 9b182fc8b3834a57a90c369bd9f582cb (table txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364])
14:07:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.386802 19025 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:07:21.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.387285 19025 catalog_manager.cc:1261] Loaded cluster ID: 1900d7fbdc174b3c9fae2ae657ce432f
14:07:21.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.387339 19025 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:07:21.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.387917 19025 catalog_manager.cc:1506] Loading token signing keys...
14:07:21.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.388353 19025 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae: Loaded TSK: 0
14:07:21.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.389223 19025 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:07:21.389 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.124:33213(127.17.164.124:33213)] connecting to peer
14:07:21.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.391 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.124:33213(127.17.164.124:33213)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.391 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.124:33213: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213
14:07:21.393 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:07:21.393 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=4), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [2ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: Network error: Failed to connect to peer master-127.17.164.124:33213(127.17.164.124:33213): Connection refused: /127.17.164.124:33213, [2ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: OK, deferred=null) after lookup
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.428846 18188 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38194:
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:07:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:07:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.430651 18188 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:07:21.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.463500 18959 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 5 FOLLOWER]: Refusing update from remote peer 3c2b262c9aae4117ae997f8710cd51ae: Log matching property violated. Preceding OpId in replica: term: 2 index: 11. Preceding OpId from leader: term: 5 index: 13. (index mismatch)
14:07:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.464342 19023 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [LEADER]: Connected to new peer: Peer: permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
14:07:21.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.467201 19038 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "3c2b262c9aae4117ae997f8710cd51ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:21.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.467346 19038 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: This master's current role is: FOLLOWER
14:07:21.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.467800 19004 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3c2b262c9aae4117ae997f8710cd51ae. Latest consensus state: current_term: 5 leader_uuid: "3c2b262c9aae4117ae997f8710cd51ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:21.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.467933 19004 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: This master's current role is: FOLLOWER
14:07:21.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.468811 19019 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "3c2b262c9aae4117ae997f8710cd51ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:21.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.468969 19019 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae [sys.catalog]: This master's current role is: LEADER
14:07:21.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.469295 19004 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "3c2b262c9aae4117ae997f8710cd51ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:21.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.469421 19004 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: This master's current role is: FOLLOWER
14:07:21.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.470110 18175 consensus_peers.cc:487] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae -> Peer 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213): Couldn't send request to peer 8e81f7c39b084eb1863771945700e98e. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:07:21.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.587986 18805 heartbeater.cc:344] Connected to a master server at 127.17.164.126:34533
14:07:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.589373 18949 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "20377aa649e94659bf7c8827077fc035" instance_seqno: 1750601238301924) as {username='slave'} at 127.17.164.67:57393; Asking this server to re-register.
14:07:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.589838 18805 heartbeater.cc:461] Registering TS with master...
14:07:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:21.590699 18949 ts_manager.cc:194] Registered new tserver with Master: 20377aa649e94659bf7c8827077fc035 (127.17.164.67:37573)
14:07:21.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:21.988933 18175 consensus_peers.cc:487] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae -> Peer 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213): Couldn't send request to peer 8e81f7c39b084eb1863771945700e98e. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:07:22.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:22.248769 18175 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111) [suppressed 14 similar messages]
14:07:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:22.457428 18155 master_runner.cc:186] Error getting master registration for 127.17.164.124:33213: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111),
14:07:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:22.457563 18155 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:22.466239 18175 consensus_peers.cc:487] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae -> Peer 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213): Couldn't send request to peer 8e81f7c39b084eb1863771945700e98e. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
14:07:22.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:22.586741 18806 heartbeater.cc:499] Master 127.17.164.125:41689 was elected leader, sending a full tablet report...
14:07:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:22.953547 18175 consensus_peers.cc:487] T 00000000000000000000000000000000 P 3c2b262c9aae4117ae997f8710cd51ae -> Peer 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213): Couldn't send request to peer 8e81f7c39b084eb1863771945700e98e. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.124:33213: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
14:07:23.144 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.164.124:33213
14:07:23.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:07:23.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.145712 18065 external_mini_cluster.cc:1351] Running /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:23.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu
14:07:23.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/logs
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data/info.pb
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.124:33213
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.124
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45341
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33855/jwks.json
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:07:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:07:23.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.156977 18664 heartbeater.cc:499] Master 127.17.164.125:41689 was elected leader, sending a full tablet report...
14:07:23.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.157655 18521 heartbeater.cc:499] Master 127.17.164.125:41689 was elected leader, sending a full tablet report...
14:07:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.271692 19060 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.271989 19060 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.272043 19060 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.275475 19060 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.275552 19060 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:07:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.275568 19060 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.275588 19060 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.275609 19060 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.275627 19060 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.275642 19060 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33855/jwks.json
14:07:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.279724 19060 master_runner.cc:386] Master server non-default flags:
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.164.84:35041
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.164.126:34533,127.17.164.125:41689,127.17.164.124:33213
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.164.124:33213
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33855/jwks.json
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data/info.pb
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.164.124
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45341
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.19060
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/logs
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:13 UTC on 24a791456cd2
14:07:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6714
14:07:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.281378 19060 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.282883 19060 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.290930 19068 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.290932 19067 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.290975 19070 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.291870 19060 server_base.cc:1048] running on GCE node
14:07:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.292610 19060 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.293753 19060 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:23.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.294907 19060 hybrid_clock.cc:648] HybridClock initialized: now 1750601243294893 us; error 36 us; skew 500 ppm
14:07:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.297227 19060 webserver.cc:469] Webserver started at http://127.17.164.124:45341/ using document root <none> and password file <none>
14:07:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.297895 19060 fs_manager.cc:362] Metadata directory not provided
14:07:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.297967 19060 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:23.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.302135 19060 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.004s sys 0.000s
14:07:23.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.304538 19076 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.305958 19060 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.306084 19060 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data,/tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e81f7c39b084eb1863771945700e98e"
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:07:17 on dist-test-slave-77bb"
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.306600 19060 fs_report.cc:389] FS layout report
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/wal
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data/data
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:23.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.334318 19060 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.335156 19060 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:23.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.335331 19060 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.344663 19060 rpc_server.cc:307] RPC server started. Bound to: 127.17.164.124:33213
14:07:23.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.344676 19138 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.164.124:33213 every 8 connection(s)
14:07:23.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.346149 19060 server_base.cc:1180] Dumped server information to /tmp/dist-test-task7ZMTbR/test-tmp/mini-kudu-cluster7465509555052795891/master-2/data/info.pb
14:07:23.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.351962 19140 sys_catalog.cc:263] Verifying existing consensus state
14:07:23.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.353278 18065 external_mini_cluster.cc:1413] Started /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu as pid 19060
14:07:23.354 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:23.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.354156 19140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: Bootstrap starting.
14:07:23.354 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.164.125:41689
14:07:23.354 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:07:23.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.354799 18065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu with pid 18155
14:07:23.366 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.17.164.125:41689(127.17.164.125:41689)] handling channelInactive
14:07:23.367 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.125:41689(127.17.164.125:41689)] cleaning up while in state READY due to: connection closed
14:07:23.369 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:23.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.377341 19140 log.cc:826] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: Log is configured to *not* fsync() on all Append() calls
14:07:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.389190 19140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:07:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.389598 19140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: Bootstrap complete.
14:07:23.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.393646 19140 raft_consensus.cc:357] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:23.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.394037 19140 raft_consensus.cc:738] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8e81f7c39b084eb1863771945700e98e, State: Initialized, Role: FOLLOWER
14:07:23.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.394574 19140 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 4.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:23.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.395100 19143 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:23.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.395258 19143 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: This master's current role is: FOLLOWER
14:07:23.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.395874 19140 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: configured and running, proceeding with master startup.
14:07:23.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.396253 19103 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 4 FOLLOWER]: Advancing to term 5
14:07:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.399736 19156 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "3c2b262c9aae4117ae997f8710cd51ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.399845 19156 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: This master's current role is: FOLLOWER
14:07:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.400379 19143 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3c2b262c9aae4117ae997f8710cd51ae. Latest consensus state: current_term: 5 leader_uuid: "3c2b262c9aae4117ae997f8710cd51ae" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.400472 19143 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: This master's current role is: FOLLOWER
14:07:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.401013 19077 connection.cc:441] server connection from 127.0.0.1:37428 torn down before Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:37428 (request call id 0) could send its response
14:07:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.404027 19157 catalog_manager.cc:1261] Loaded cluster ID: 1900d7fbdc174b3c9fae2ae657ce432f
14:07:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.404089 19157 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: loading cluster ID for follower catalog manager: success
14:07:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.406582 19157 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: acquiring CA information for follower catalog manager: success
14:07:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.408269 19157 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:07:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.409032 19060 master_runner.cc:186] Error getting master registration for 127.17.164.125:41689: Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111),
14:07:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.410687 19060 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:23.510 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.125:41689(127.17.164.125:41689)] connecting to peer
14:07:23.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.125:41689(127.17.164.125:41689)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.511 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.17.164.125:41689(127.17.164.125:41689) for tablet Kudu Master: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.124:33213(127.17.164.124:33213)] connecting to peer
14:07:23.530 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.125:41689(127.17.164.125:41689)] connecting to peer
14:07:23.531 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.17.164.124:33213(127.17.164.124:33213)] Successfully connected to peer
14:07:23.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.125:41689(127.17.164.125:41689)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.531 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.125:41689: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.542 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x51242056, L:/127.0.0.1:37442 - R:/127.17.164.124:33213] using TOKEN/null
14:07:23.544 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:23.549 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.125:41689(127.17.164.125:41689)] connecting to peer
14:07:23.550 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.550 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.125:41689(127.17.164.125:41689)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.125:41689: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.551 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:23.569 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.125:41689(127.17.164.125:41689)] connecting to peer
14:07:23.570 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.570 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.125:41689(127.17.164.125:41689)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.125:41689: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:23.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.582288 19161 raft_consensus.cc:491] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 3c2b262c9aae4117ae997f8710cd51ae)
14:07:23.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.582449 19161 raft_consensus.cc:513] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.583729 19161 leader_election.cc:290] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689), 8e81f7c39b084eb1863771945700e98e (127.17.164.124:33213)
14:07:23.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.584518 18935 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111)
14:07:23.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.585557 18935 leader_election.cc:336] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689): Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111)
14:07:23.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.587945 19103 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "68e4a681ed9d4cc6bb513c955989849f" candidate_term: 6 candidate_status { last_received { term: 5 index: 13 } } ignore_live_leader: false dest_uuid: "8e81f7c39b084eb1863771945700e98e" is_pre_election: true
14:07:23.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.588603 18935 leader_election.cc:304] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 68e4a681ed9d4cc6bb513c955989849f; no voters: 3c2b262c9aae4117ae997f8710cd51ae, 8e81f7c39b084eb1863771945700e98e
14:07:23.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.588821 19161 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
14:07:23.589 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.125:41689(127.17.164.125:41689)] connecting to peer
14:07:23.590 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.125:41689(127.17.164.125:41689)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.591 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.125:41689: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.591 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:23.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.594791 18804 heartbeater.cc:344] Connected to a master server at 127.17.164.124:33213
14:07:23.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.596545 19093 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "20377aa649e94659bf7c8827077fc035" instance_seqno: 1750601238301924) as {username='slave'} at 127.17.164.67:60495; Asking this server to re-register.
14:07:23.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.597024 18804 heartbeater.cc:461] Registering TS with master...
14:07:23.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.598133 19093 ts_manager.cc:194] Registered new tserver with Master: 20377aa649e94659bf7c8827077fc035 (127.17.164.67:37573)
14:07:23.609 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.125:41689(127.17.164.125:41689)] connecting to peer
14:07:23.610 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.610 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.125:41689(127.17.164.125:41689)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.610 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.125:41689: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.611 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:23.690 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.125:41689(127.17.164.125:41689)] connecting to peer
14:07:23.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.125:41689(127.17.164.125:41689)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.691 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.125:41689: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.691 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689; will retry
14:07:23.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.699337 19143 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 3c2b262c9aae4117ae997f8710cd51ae)
14:07:23.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.699555 19143 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:23.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.700868 19143 leader_election.cc:290] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689)
14:07:23.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.701398 18959 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8e81f7c39b084eb1863771945700e98e" candidate_term: 6 candidate_status { last_received { term: 5 index: 13 } } ignore_live_leader: false dest_uuid: "68e4a681ed9d4cc6bb513c955989849f" is_pre_election: true
14:07:23.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.701484 19079 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111)
14:07:23.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.701591 18959 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8e81f7c39b084eb1863771945700e98e in term 5.
14:07:23.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.701963 19077 leader_election.cc:304] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 68e4a681ed9d4cc6bb513c955989849f, 8e81f7c39b084eb1863771945700e98e; no voters:
14:07:23.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.702200 19143 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 5 FOLLOWER]: Leader pre-election won for term 6
14:07:23.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.702265 19143 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 5 FOLLOWER]: Starting leader election (detected failure of leader 3c2b262c9aae4117ae997f8710cd51ae)
14:07:23.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.702308 19143 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 5 FOLLOWER]: Advancing to term 6
14:07:23.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.702422 19079 leader_election.cc:336] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689): Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111)
14:07:23.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.703289 19143 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:23.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.703627 19143 leader_election.cc:290] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 6 election: Requested vote from peers 68e4a681ed9d4cc6bb513c955989849f (127.17.164.126:34533), 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689)
14:07:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.703984 18959 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8e81f7c39b084eb1863771945700e98e" candidate_term: 6 candidate_status { last_received { term: 5 index: 13 } } ignore_live_leader: false dest_uuid: "68e4a681ed9d4cc6bb513c955989849f"
14:07:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.704183 18959 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 5 FOLLOWER]: Advancing to term 6
14:07:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.704650 19079 leader_election.cc:336] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689): Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111)
14:07:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.705214 18959 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8e81f7c39b084eb1863771945700e98e in term 6.
14:07:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.705489 19077 leader_election.cc:304] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 68e4a681ed9d4cc6bb513c955989849f, 8e81f7c39b084eb1863771945700e98e; no voters: 3c2b262c9aae4117ae997f8710cd51ae
14:07:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.705658 19143 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 6 FOLLOWER]: Leader election won for term 6
14:07:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.705762 19143 raft_consensus.cc:695] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [term 6 LEADER]: Becoming Leader. State: Replica: 8e81f7c39b084eb1863771945700e98e, State: Running, Role: LEADER
14:07:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.706095 19143 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 5.13, Last appended by leader: 13, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } }
14:07:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.707058 19156 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8e81f7c39b084eb1863771945700e98e. Latest consensus state: current_term: 6 leader_uuid: "8e81f7c39b084eb1863771945700e98e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.707177 19156 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: This master's current role is: LEADER
14:07:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.707518 19168 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:07:23.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.709136 19168 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=219e1776ae3e40c18439e295cda92e80]
14:07:23.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.709492 19168 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364]
14:07:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.710419 19168 tablet_loader.cc:96] loaded metadata for tablet 5df4ed739ba24610b20e8d8bfc9616d8 (table kudu_system.kudu_transactions [id=219e1776ae3e40c18439e295cda92e80])
14:07:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.710736 19168 tablet_loader.cc:96] loaded metadata for tablet 991f2c43aa654bb9a26fb5f15304e043 (table txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364])
14:07:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.710988 19168 tablet_loader.cc:96] loaded metadata for tablet 9b182fc8b3834a57a90c369bd9f582cb (table txn_manager_fallback_rolling [id=93e6eaae358242959886acaf4bb50364])
14:07:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.711104 19168 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:07:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.711491 19168 catalog_manager.cc:1261] Loaded cluster ID: 1900d7fbdc174b3c9fae2ae657ce432f
14:07:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.711542 19168 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:07:23.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.712148 19168 catalog_manager.cc:1506] Loading token signing keys...
14:07:23.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.712544 19168 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e: Loaded TSK: 0
14:07:23.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.713263 19168 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:07:23.749 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.17.164.125:41689(127.17.164.125:41689)] connecting to peer
14:07:23.750 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.750 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.125:41689(127.17.164.125:41689)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.164.125:41689: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689
14:07:23.751 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:07:23.751 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] sending RPC to server master-127.17.164.125:41689, [2ms] received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [22ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [23ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [35ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [35ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [61ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [82ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [101ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [102ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [182ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [182ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [241ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [241ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, deferred=Deferred@835184360(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@73ca36d4 -> (continuation of Deferred@430683354 after retry RPC after error@2083712549) -> (continuation of Deferred@1917630445 after retry RPC after error@1343374310) -> (continuation of Deferred@341520474 after retry RPC after error@2146674482) -> (continuation of Deferred@1754760446 after retry RPC after error@718858105) -> (continuation of Deferred@1733086104 after retry RPC after error@90450548) -> (continuation of Deferred@1367404114 after retry RPC after error@1007560756), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@3d9f3426 -> (continuation of Deferred@430683354 after retry RPC after error@2083712549) -> (continuation of Deferred@1917630445 after retry RPC after error@1343374310) -> (continuation of Deferred@341520474 after retry RPC after error@2146674482) -> (continuation of Deferred@1754760446 after retry RPC after error@718858105) -> (continuation of Deferred@1733086104 after retry RPC after error@90450548) -> (continuation of Deferred@1367404114 after retry RPC after error@1007560756))) after lookup
14:07:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.761107 19123 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:37442 (request call id 7) took 8 ms (client timeout 8 ms). Trace:
14:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.761215 19123 rpcz_store.cc:269] 0622 14:07:23.752800 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:07:23.752894 (+ 94us) service_pool.cc:225] Handling call
14:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:07:23.761063 (+ 8169us) inbound_call.cc:173] Queueing success response
14:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":72,"init.queue_time_us":68,"thread_start_us":346,"threads_started":4}
14:07:23.763 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=253), Traces: [0ms] sending RPC to server master-127.17.164.125:41689, [2ms] received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [22ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [23ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [35ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [35ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [61ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [82ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [101ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [102ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [182ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [182ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [241ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [241ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [243ms] sending RPC to server master-127.17.164.124:33213, [253ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@835184360(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@73ca36d4 -> (continuation of Deferred@430683354 after retry RPC after error@2083712549) -> (continuation of Deferred@1917630445 after retry RPC after error@1343374310) -> (continuation of Deferred@341520474 after retry RPC after error@2146674482) -> (continuation of Deferred@1754760446 after retry RPC after error@718858105) -> (continuation of Deferred@1733086104 after retry RPC after error@90450548) -> (continuation of Deferred@1367404114 after retry RPC after error@1007560756) -> (continuation of Deferred@428645995 after retry RPC@1328021878), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@3d9f3426 -> (continuation of Deferred@430683354 after retry RPC after error@2083712549) -> (continuation of Deferred@1917630445 after retry RPC after error@1343374310) -> (continuation of Deferred@341520474 after retry RPC after error@2146674482) -> (continuation of Deferred@1754760446 after retry RPC after error@718858105) -> (continuation of Deferred@1733086104 after retry RPC after error@90450548) -> (continuation of Deferred@1367404114 after retry RPC after error@1007560756) -> (continuation of Deferred@428645995 after retry RPC@1328021878)))
14:07:23.764 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=253), Traces: [0ms] sending RPC to server master-127.17.164.125:41689, [2ms] received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [22ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [23ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [35ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [35ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [41ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [61ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [62ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [82ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [101ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [102ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [182ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [182ms] delaying RPC due to: Service unavailable: Master config (127.17.164.124:33213,127.17.164.126:34533,127.17.164.125:41689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.124:33213, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.126:34533, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.164.125:41689, [241ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.125:41689: Network error: Failed to connect to peer master-127.17.164.125:41689(127.17.164.125:41689): Connection refused: /127.17.164.125:41689, [241ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.124:33213: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.17.164.126:34533: OK, [243ms] sending RPC to server master-127.17.164.124:33213, [253ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@835184360(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@73ca36d4 -> (continuation of Deferred@430683354 after retry RPC after error@2083712549) -> (continuation of Deferred@1917630445 after retry RPC after error@1343374310) -> (continuation of Deferred@341520474 after retry RPC after error@2146674482) -> (continuation of Deferred@1754760446 after retry RPC after error@718858105) -> (continuation of Deferred@1733086104 after retry RPC after error@90450548) -> (continuation of Deferred@1367404114 after retry RPC after error@1007560756) -> (continuation of Deferred@428645995 after retry RPC@1328021878), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@3d9f3426 -> (continuation of Deferred@430683354 after retry RPC after error@2083712549) -> (continuation of Deferred@1917630445 after retry RPC after error@1343374310) -> (continuation of Deferred@341520474 after retry RPC after error@2146674482) -> (continuation of Deferred@1754760446 after retry RPC after error@718858105) -> (continuation of Deferred@1733086104 after retry RPC after error@90450548) -> (continuation of Deferred@1367404114 after retry RPC after error@1007560756) -> (continuation of Deferred@428645995 after retry RPC@1328021878)))
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.792901 19093 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:37450:
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:07:23.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.795504 19140 master.cc:464] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 1 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
14:07:23.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.796391 18959 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [term 6 FOLLOWER]: Refusing update from remote peer 8e81f7c39b084eb1863771945700e98e: Log matching property violated. Preceding OpId in replica: term: 5 index: 13. Preceding OpId from leader: term: 6 index: 14. (index mismatch)
14:07:23.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.796872 19156 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [LEADER]: Connected to new peer: Peer: permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
14:07:23.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.798174 19161 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8e81f7c39b084eb1863771945700e98e. Latest consensus state: current_term: 6 leader_uuid: "8e81f7c39b084eb1863771945700e98e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:23.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.798312 19161 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: This master's current role is: FOLLOWER
14:07:23.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.799556 19156 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 6 leader_uuid: "8e81f7c39b084eb1863771945700e98e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:23.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.799674 19156 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e [sys.catalog]: This master's current role is: LEADER
14:07:23.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.800004 19161 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 6 leader_uuid: "8e81f7c39b084eb1863771945700e98e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68e4a681ed9d4cc6bb513c955989849f" member_type: VOTER last_known_addr { host: "127.17.164.126" port: 34533 } } peers { permanent_uuid: "3c2b262c9aae4117ae997f8710cd51ae" member_type: VOTER last_known_addr { host: "127.17.164.125" port: 41689 } } peers { permanent_uuid: "8e81f7c39b084eb1863771945700e98e" member_type: VOTER last_known_addr { host: "127.17.164.124" port: 33213 } } }
14:07:23.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:23.800141 19161 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68e4a681ed9d4cc6bb513c955989849f [sys.catalog]: This master's current role is: FOLLOWER
14:07:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:23.800879 19079 consensus_peers.cc:487] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e -> Peer 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689): Couldn't send request to peer 3c2b262c9aae4117ae997f8710cd51ae. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:07:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.020253 19123 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:37442 (request call id 8) took 250 ms (client timeout 250 ms). Trace:
14:07:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.020339 19123 rpcz_store.cc:269] 0622 14:07:23.770136 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:07:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:07:23.770222 (+ 86us) service_pool.cc:225] Handling call
14:07:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:07:24.020231 (+250009us) inbound_call.cc:173] Queueing success response
14:07:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:07:24.021 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.17.164.124:33213, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@36364079(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@4650f548, errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@3bd2542f))
14:07:24.021 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.17.164.124:33213, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@36364079(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@4650f548, errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@3bd2542f))
14:07:24.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.054224 18688 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.058s from last keepalive heartbeat (effective timeout is 1.000s)
14:07:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.059187 19179 tablet_replica.cc:1304] Aborted: operation has been aborted: cancelling pending write operations
14:07:24.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.163266 18519 heartbeater.cc:344] Connected to a master server at 127.17.164.124:33213
14:07:24.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.163411 18519 heartbeater.cc:499] Master 127.17.164.124:33213 was elected leader, sending a full tablet report...
14:07:24.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.163919 18662 heartbeater.cc:344] Connected to a master server at 127.17.164.124:33213
14:07:24.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.164052 18662 heartbeater.cc:499] Master 127.17.164.124:33213 was elected leader, sending a full tablet report...
14:07:24.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.164469 19092 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "20ee773143da465dbdafbd4e4afd6a7b" instance_seqno: 1750601237873680) as {username='slave'} at 127.17.164.65:52557; Asking this server to re-register.
14:07:24.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.164820 19093 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a8cc3e270a1c4a4e96cb367ff07c7c68" instance_seqno: 1750601238080687) as {username='slave'} at 127.17.164.66:53723; Asking this server to re-register.
14:07:24.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.164937 18519 heartbeater.cc:461] Registering TS with master...
14:07:24.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.165125 18519 heartbeater.cc:507] Master 127.17.164.124:33213 requested a full tablet report, sending...
14:07:24.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.165176 18662 heartbeater.cc:461] Registering TS with master...
14:07:24.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.165303 18662 heartbeater.cc:507] Master 127.17.164.124:33213 requested a full tablet report, sending...
14:07:24.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.166069 19092 ts_manager.cc:194] Registered new tserver with Master: a8cc3e270a1c4a4e96cb367ff07c7c68 (127.17.164.66:45229)
14:07:24.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.166169 19093 ts_manager.cc:194] Registered new tserver with Master: 20ee773143da465dbdafbd4e4afd6a7b (127.17.164.65:37241)
14:07:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.280333 19123 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:37442 (request call id 9) took 250 ms (client timeout 250 ms). Trace:
14:07:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.280443 19123 rpcz_store.cc:269] 0622 14:07:24.030145 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:07:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:07:24.030250 (+ 105us) service_pool.cc:225] Handling call
14:07:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:07:24.280298 (+250048us) inbound_call.cc:173] Queueing success response
14:07:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:07:24.281 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.17.164.124:33213, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@566419559(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@818bb5c, errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@4a28899c))
14:07:24.281 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.17.164.124:33213, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@566419559(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@818bb5c, errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@4a28899c))
14:07:24.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.348256 19079 consensus_peers.cc:487] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e -> Peer 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689): Couldn't send request to peer 3c2b262c9aae4117ae997f8710cd51ae. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.540289 19123 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:37442 (request call id 10) took 250 ms (client timeout 250 ms). Trace:
14:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.540443 19123 rpcz_store.cc:269] 0622 14:07:24.289986 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:07:24.290073 (+ 87us) service_pool.cc:225] Handling call
14:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:07:24.540238 (+250165us) inbound_call.cc:173] Queueing success response
14:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:07:24.541 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.17.164.124:33213, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1742798407(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@5117eb04, errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@1c987821))
14:07:24.541 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=252), Traces: [0ms] sending RPC to server master-127.17.164.124:33213, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1742798407(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@5117eb04, errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@1c987821))
14:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.800197 19123 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:37442 (request call id 11) took 250 ms (client timeout 250 ms). Trace:
14:07:24.801 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.17.164.124:33213, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@309491271(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@7200b088, errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@1ca12a07))
14:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.800313 19123 rpcz_store.cc:269] 0622 14:07:24.549903 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:07:24.549992 (+ 89us) service_pool.cc:225] Handling call
14:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:07:24.800165 (+250173us) inbound_call.cc:173] Queueing success response
14:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:07:24.801 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.17.164.124:33213, [252ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@309491271(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/1814209607@7200b088, errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/224862967@1ca12a07))
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:24.806859 19092 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:37466:
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.807539 19092 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:07:24.820 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 20377aa649e94659bf7c8827077fc035 (127.17.164.67:37573): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1750601238 last_transition_timestamp: 1750601244
14:07:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:24.899081 19079 consensus_peers.cc:487] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e -> Peer 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689): Couldn't send request to peer 3c2b262c9aae4117ae997f8710cd51ae. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
14:07:25.374 [DEBUG - main] (Connection.java:575) [peer master-127.17.164.126:34533(127.17.164.126:34533)] disconnecting while in state READY
14:07:25.374 [DEBUG - main] (Connection.java:575) [peer master-127.17.164.125:41689(127.17.164.125:41689)] disconnecting while in state TERMINATED
14:07:25.374 [DEBUG - main] (Connection.java:575) [peer 20377aa649e94659bf7c8827077fc035(127.17.164.67:37573)] disconnecting while in state READY
14:07:25.374 [DEBUG - main] (Connection.java:575) [peer master-127.17.164.124:33213(127.17.164.124:33213)] disconnecting while in state READY
14:07:25.375 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.17.164.126:34533(127.17.164.126:34533)] handling channelInactive
14:07:25.375 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.164.126:34533(127.17.164.126:34533)] cleaning up while in state READY due to: connection closed
14:07:25.375 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:07:25.376 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 20377aa649e94659bf7c8827077fc035(127.17.164.67:37573)] handling channelInactive
14:07:25.376 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 20377aa649e94659bf7c8827077fc035(127.17.164.67:37573)] cleaning up while in state READY due to: connection closed
14:07:25.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:25.376770 18065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu with pid 18382
14:07:25.376 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.17.164.124:33213(127.17.164.124:33213)] handling channelInactive
14:07:25.377 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.164.124:33213(127.17.164.124:33213)] cleaning up while in state READY due to: connection closed
14:07:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:25.384280 18065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu with pid 18525
14:07:25.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:25.391278 18065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu with pid 18667
14:07:25.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:07:25.396395 19079 consensus_peers.cc:487] T 00000000000000000000000000000000 P 8e81f7c39b084eb1863771945700e98e -> Peer 3c2b262c9aae4117ae997f8710cd51ae (127.17.164.125:41689): Couldn't send request to peer 3c2b262c9aae4117ae997f8710cd51ae. Status: Network error: Client connection negotiation failed: client connection to 127.17.164.125:41689: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
14:07:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:25.400380 18065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu with pid 18918
14:07:25.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:07:25.408093 18065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task7ZMTbR/build/debug/bin/kudu with pid 19060
14:07:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-22T14:07:25Z chronyd exiting
14:07:25.466 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)