21:03:37.632 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:03:37.632 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task6gQn2r/test-tmp
21:03:37.632 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
21:03:37.633 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task6gQn2r/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
21:03:37.634 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:03:37.954 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:03:37.954 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:03:37.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:37.956836 31749 mini_oidc.cc:150] Starting JWKS server
21:03:37.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:37.959121 31749 webserver.cc:492] Webserver started at http://127.0.0.1:32943/ using document root <none> and password file <none>
21:03:37.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:37.959398 31749 mini_oidc.cc:186] Starting OIDC Discovery server
21:03:37.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:37.959947 31749 webserver.cc:492] Webserver started at http://127.0.0.1:35313/ using document root <none> and password file <none>
21:03:37.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-14T21:03:37Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:03:37.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-14T21:03:37Z Disabled control of system clock
21:03:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:37.987484 31749 external_mini_cluster.cc:1366] Running /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/wal
21:03:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/data
21:03:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/logs
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/data/info.pb
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.126:41315
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.126
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32943/jwks.json
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:03:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:03:38.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.302279 31760 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:38.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.302577 31760 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:38.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.302654 31760 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:38.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.310588 31760 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:03:38.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.310683 31760 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:03:38.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.310729 31760 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:38.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.310767 31760 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:38.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.310818 31760 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:03:38.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.310856 31760 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:38.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.310897 31760 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32943/jwks.json
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.320406 31760 master_runner.cc:386] Master server non-default flags:
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/data
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/wal
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.126:41315
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32943/jwks.json
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/data/info.pb
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.126
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/logs
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:23 UTC on 24a791456cd2
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9516
21:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:38.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.322103 31760 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.323803 31760 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.331753 31766 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.332165 31768 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.331686 31765 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.333103 31760 server_base.cc:1047] running on GCE node
21:03:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.333818 31760 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.335150 31760 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:38.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.336417 31760 hybrid_clock.cc:648] HybridClock initialized: now 1765746218336323 us; error 39 us; skew 500 ppm
21:03:38.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.336808 31760 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:38.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.339028 31760 webserver.cc:492] Webserver started at http://127.31.1.126:46507/ using document root <none> and password file <none>
21:03:38.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.339740 31760 fs_manager.cc:362] Metadata directory not provided
21:03:38.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.339875 31760 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:38.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.340241 31760 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.342816 31760 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/data/instance:
21:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a522657a17a4489b27da69a1907160c"
21:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:38 on dist-test-slave-2vjd"
21:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.343580 31760 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/wal/instance:
21:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a522657a17a4489b27da69a1907160c"
21:03:38.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:38 on dist-test-slave-2vjd"
21:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.348699 31760 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
21:03:38.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.351989 31774 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:38.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.353269 31760 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.003s
21:03:38.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.353430 31760 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/data,/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/wal
21:03:38.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a522657a17a4489b27da69a1907160c"
21:03:38.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:38 on dist-test-slave-2vjd"
21:03:38.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.353619 31760 fs_report.cc:389] FS layout report
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/wal
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/wal
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/data/data
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.392926 31760 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:38.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.393843 31760 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:38.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.394153 31760 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:38.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.417155 31760 rpc_server.cc:307] RPC server started. Bound to: 127.31.1.126:41315
21:03:38.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.417193 31835 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.1.126:41315 every 8 connection(s)
21:03:38.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.418692 31760 server_base.cc:1179] Dumped server information to /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/data/info.pb
21:03:38.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.422808 31836 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:38.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.424306 31749 external_mini_cluster.cc:1428] Started /tmp/dist-test-task6gQn2r/build/asan/bin/kudu as pid 31760
21:03:38.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.424638 31749 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-0/wal/instance
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.426867 31749 external_mini_cluster.cc:1366] Running /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/wal
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/data
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/logs
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/data/info.pb
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.125:33977
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.125
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32943/jwks.json
21:03:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:03:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:03:38.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.428494 31836 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } has no permanent_uuid. Determining permanent_uuid...
21:03:38.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.444211 31836 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } has no permanent_uuid. Determining permanent_uuid...
21:03:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.445341 31778 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.1.125:33977: connect: Connection refused (error 111)
21:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.446957 31836 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.125:33977: Network error: Client connection negotiation failed: client connection to 127.31.1.125:33977: connect: Connection refused (error 111)
21:03:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.496271 31836 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } attempt: 1
21:03:38.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.498078 31836 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.125:33977: Network error: Client connection negotiation failed: client connection to 127.31.1.125:33977: connect: Connection refused (error 111)
21:03:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.566254 31836 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } attempt: 2
21:03:38.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.568778 31836 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.125:33977: Network error: Client connection negotiation failed: client connection to 127.31.1.125:33977: connect: Connection refused (error 111)
21:03:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.660032 31836 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } attempt: 3
21:03:38.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.662004 31836 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.125:33977: Network error: Client connection negotiation failed: client connection to 127.31.1.125:33977: connect: Connection refused (error 111)
21:03:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.751183 31838 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.751479 31838 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.751555 31838 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.759580 31838 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:03:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.759689 31838 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:03:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.759737 31838 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.759774 31838 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.759824 31838 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:03:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.759861 31838 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.759902 31838 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32943/jwks.json
21:03:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.769527 31838 master_runner.cc:386] Master server non-default flags:
21:03:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:03:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/data
21:03:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/wal
21:03:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.125:33977
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32943/jwks.json
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/data/info.pb
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.125
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/logs
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:23 UTC on 24a791456cd2
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9516
21:03:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.771116 31838 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.772789 31838 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:38.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.780514 31847 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:38.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.780546 31846 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:38.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.780668 31849 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:38.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.781824 31838 server_base.cc:1047] running on GCE node
21:03:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.782559 31838 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.783972 31838 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.785181 31838 hybrid_clock.cc:648] HybridClock initialized: now 1765746218785128 us; error 26 us; skew 500 ppm
21:03:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.785516 31838 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.787549 31838 webserver.cc:492] Webserver started at http://127.31.1.125:44409/ using document root <none> and password file <none>
21:03:38.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.788290 31838 fs_manager.cc:362] Metadata directory not provided
21:03:38.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.788414 31838 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:38.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.788861 31838 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:03:38.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.791133 31838 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/data/instance:
21:03:38.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bf55660ec6046509a1e498a52e393e5"
21:03:38.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:38 on dist-test-slave-2vjd"
21:03:38.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.791862 31838 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/wal/instance:
21:03:38.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bf55660ec6046509a1e498a52e393e5"
21:03:38.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:38 on dist-test-slave-2vjd"
21:03:38.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.796967 31838 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
21:03:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.800161 31855 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.801388 31838 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
21:03:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.801556 31838 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/data,/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/wal
21:03:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bf55660ec6046509a1e498a52e393e5"
21:03:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:38 on dist-test-slave-2vjd"
21:03:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.801740 31838 fs_report.cc:389] FS layout report
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/wal
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/wal
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/data/data
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.805253 31836 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } attempt: 4
21:03:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.807092 31836 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.125:33977: Network error: Client connection negotiation failed: client connection to 127.31.1.125:33977: connect: Connection refused (error 111)
21:03:38.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.817137 31838 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:38.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.817865 31838 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:38.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.818111 31838 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:38.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.841208 31838 rpc_server.cc:307] RPC server started. Bound to: 127.31.1.125:33977
21:03:38.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.841250 31916 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.1.125:33977 every 8 connection(s)
21:03:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.842717 31838 server_base.cc:1179] Dumped server information to /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/data/info.pb
21:03:38.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.844561 31749 external_mini_cluster.cc:1428] Started /tmp/dist-test-task6gQn2r/build/asan/bin/kudu as pid 31838
21:03:38.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.844754 31749 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-1/wal/instance
21:03:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.845737 31749 external_mini_cluster.cc:1366] Running /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/wal
21:03:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/data
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/logs
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/data/info.pb
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.124:41201
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.124
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32943/jwks.json
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:03:38.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.846868 31917 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:38.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.854691 31917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } has no permanent_uuid. Determining permanent_uuid...
21:03:38.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.867841 31917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } has no permanent_uuid. Determining permanent_uuid...
21:03:38.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.876194 31917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } has no permanent_uuid. Determining permanent_uuid...
21:03:38.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.877305 31857 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:38.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.878862 31917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.124:41201: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:38.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.928182 31917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } attempt: 1
21:03:38.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:38.930629 31917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.124:41201: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:38.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:38.998903 31917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } attempt: 2
21:03:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.001083 31917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.124:41201: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:39.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.092290 31917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } attempt: 3
21:03:39.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.094236 31917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.124:41201: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:39.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.106323 31836 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } attempt: 5
21:03:39.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.112354 31836 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } has no permanent_uuid. Determining permanent_uuid...
21:03:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.114123 31836 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.124:41201: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:39.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.165360 31836 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } attempt: 1
21:03:39.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.167266 31836 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.124:41201: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:39.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.173735 31919 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:39.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.174005 31919 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:39.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.174083 31919 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.182296 31919 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.182402 31919 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.182487 31919 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.182530 31919 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.182580 31919 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.182618 31919 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.182660 31919 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32943/jwks.json
21:03:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.192485 31919 master_runner.cc:386] Master server non-default flags:
21:03:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:03:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/data
21:03:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/wal
21:03:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.124:41201
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32943/jwks.json
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/data/info.pb
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.124
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/logs
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:23 UTC on 24a791456cd2
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9516
21:03:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.194116 31919 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.195811 31919 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.203469 31928 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.203513 31927 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.204528 31930 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.204592 31919 server_base.cc:1047] running on GCE node
21:03:39.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.205375 31919 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:39.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.206734 31919 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:39.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.207988 31919 hybrid_clock.cc:648] HybridClock initialized: now 1765746219207931 us; error 30 us; skew 500 ppm
21:03:39.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.208338 31919 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.210589 31919 webserver.cc:492] Webserver started at http://127.31.1.124:42021/ using document root <none> and password file <none>
21:03:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.211282 31919 fs_manager.cc:362] Metadata directory not provided
21:03:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.211407 31919 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.211759 31919 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:03:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.214192 31919 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/data/instance:
21:03:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24794a00822242cba0e345d7ec9fd95f"
21:03:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:39 on dist-test-slave-2vjd"
21:03:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.214958 31919 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/wal/instance:
21:03:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24794a00822242cba0e345d7ec9fd95f"
21:03:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:39 on dist-test-slave-2vjd"
21:03:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.220134 31919 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
21:03:39.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.223417 31936 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.224611 31919 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.224793 31919 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/data,/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/wal
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24794a00822242cba0e345d7ec9fd95f"
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:39 on dist-test-slave-2vjd"
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.224973 31919 fs_report.cc:389] FS layout report
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/wal
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/wal
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/data/data
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.235491 31836 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } attempt: 2
21:03:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.237219 31836 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.124:41201: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.237462 31917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } attempt: 4
21:03:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.238996 31917 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.1.124:41201: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:39.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.245491 31919 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:39.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.246202 31919 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:39.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.246433 31919 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.268447 31919 rpc_server.cc:307] RPC server started. Bound to: 127.31.1.124:41201
21:03:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.268488 31997 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.1.124:41201 every 8 connection(s)
21:03:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.269922 31919 server_base.cc:1179] Dumped server information to /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/data/info.pb
21:03:39.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.274217 31998 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:39.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.274219 31749 external_mini_cluster.cc:1428] Started /tmp/dist-test-task6gQn2r/build/asan/bin/kudu as pid 31919
21:03:39.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.274401 31749 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/master-2/wal/instance
21:03:39.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.281248 31998 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } has no permanent_uuid. Determining permanent_uuid...
21:03:39.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.293443 31998 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } has no permanent_uuid. Determining permanent_uuid...
21:03:39.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.301175 31998 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } has no permanent_uuid. Determining permanent_uuid...
21:03:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.312333 31998 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f: Bootstrap starting.
21:03:39.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.315541 31998 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f: Neither blocks nor log segments found. Creating new log.
21:03:39.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.316488 31998 log.cc:826] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f: Log is configured to *not* fsync() on all Append() calls
21:03:39.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.318843 31998 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f: No bootstrap required, opened a new log
21:03:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.323912 31998 raft_consensus.cc:359] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.324175 31998 raft_consensus.cc:385] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.324257 31998 raft_consensus.cc:740] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 24794a00822242cba0e345d7ec9fd95f, State: Initialized, Role: FOLLOWER
21:03:39.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.324839 31998 consensus_queue.cc:260] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:39.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.325636 32007 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:39.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.325870 32007 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.326874 31998 sys_catalog.cc:565] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [sys.catalog]: configured and running, proceeding with master startup.
21:03:39.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.336143 32018 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:03:39.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.336277 32018 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:03:39.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.333571 31919 master_runner.cc:186] Error getting master registration for 127.31.1.126:41315: OK, instance_id { permanent_uuid: "0a522657a17a4489b27da69a1907160c" instance_seqno: 1765746218411811 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:03:39.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.339289 31919 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:03:39.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.343446 31836 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } attempt: 3
21:03:39.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.355790 31836 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c: Bootstrap starting.
21:03:39.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.356276 32007 raft_consensus.cc:493] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:39.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.356559 32007 raft_consensus.cc:515] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:39.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.358361 32007 leader_election.cc:290] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0a522657a17a4489b27da69a1907160c (127.31.1.126:41315), 6bf55660ec6046509a1e498a52e393e5 (127.31.1.125:33977)
21:03:39.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.359957 31836 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c: Neither blocks nor log segments found. Creating new log.
21:03:39.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.361372 31836 log.cc:826] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c: Log is configured to *not* fsync() on all Append() calls
21:03:39.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.364476 31836 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c: No bootstrap required, opened a new log
21:03:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.372304 31836 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.372642 31836 raft_consensus.cc:385] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.372788 31836 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0a522657a17a4489b27da69a1907160c, State: Initialized, Role: FOLLOWER
21:03:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.373370 31836 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.374310 32021 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.374606 32021 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.375373 31836 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [sys.catalog]: configured and running, proceeding with master startup.
21:03:39.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.375438 31801 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24794a00822242cba0e345d7ec9fd95f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0a522657a17a4489b27da69a1907160c" is_pre_election: true
21:03:39.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.375909 31801 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24794a00822242cba0e345d7ec9fd95f in term 0.
21:03:39.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.376619 31938 leader_election.cc:304] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0a522657a17a4489b27da69a1907160c, 24794a00822242cba0e345d7ec9fd95f; no voters:
21:03:39.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.377106 32007 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [term 0 FOLLOWER]: Leader pre-election won for term 1
21:03:39.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.377300 32007 raft_consensus.cc:493] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:39.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.377421 32007 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [term 0 FOLLOWER]: Advancing to term 1
21:03:39.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.382547 32007 raft_consensus.cc:515] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:39.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.383630 32007 leader_election.cc:290] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [CANDIDATE]: Term 1 election: Requested vote from peers 0a522657a17a4489b27da69a1907160c (127.31.1.126:41315), 6bf55660ec6046509a1e498a52e393e5 (127.31.1.125:33977)
21:03:39.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.384498 31801 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24794a00822242cba0e345d7ec9fd95f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0a522657a17a4489b27da69a1907160c"
21:03:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.385030 31801 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 0 FOLLOWER]: Advancing to term 1
21:03:39.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.390941 31801 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 24794a00822242cba0e345d7ec9fd95f in term 1.
21:03:39.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.393173 31938 leader_election.cc:304] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0a522657a17a4489b27da69a1907160c, 24794a00822242cba0e345d7ec9fd95f; no voters:
21:03:39.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.393594 32007 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [term 1 FOLLOWER]: Leader election won for term 1
21:03:39.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.396924 31780 tablet.cc:2378] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:39.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.397680 32032 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:03:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:39.397933 32032 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:03:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.398206 32007 raft_consensus.cc:697] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [term 1 LEADER]: Becoming Leader. State: Replica: 24794a00822242cba0e345d7ec9fd95f, State: Running, Role: LEADER
21:03:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.398775 32007 consensus_queue.cc:237] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.397063 31760 master_runner.cc:186] Error getting master registration for 127.31.1.125:33977: OK, instance_id { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" instance_seqno: 1765746218835645 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:03:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.405987 31760 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:03:39.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.410583 32033 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24794a00822242cba0e345d7ec9fd95f. Latest consensus state: current_term: 1 leader_uuid: "24794a00822242cba0e345d7ec9fd95f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:39.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.410902 32033 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [sys.catalog]: This master's current role is: LEADER
21:03:39.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.423362 32036 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:03:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.426003 32036 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:03:39.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.466202 31801 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 1 FOLLOWER]: Refusing update from remote peer 24794a00822242cba0e345d7ec9fd95f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:03:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.467142 32033 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:39.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.482324 32021 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24794a00822242cba0e345d7ec9fd95f. Latest consensus state: current_term: 1 leader_uuid: "24794a00822242cba0e345d7ec9fd95f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.482681 32021 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.488859 32037 mvcc.cc:204] Tried to move back new op lower bound from 7232496514919968768 to 7232496514662694912. Current Snapshot: MvccSnapshot[applied={T|T < 7232496514919968768}]
21:03:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.489552 32038 mvcc.cc:204] Tried to move back new op lower bound from 7232496514919968768 to 7232496514662694912. Current Snapshot: MvccSnapshot[applied={T|T < 7232496514919968768}]
21:03:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.498713 32021 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "24794a00822242cba0e345d7ec9fd95f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.498961 32021 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.505156 32036 catalog_manager.cc:1357] Generated new cluster ID: 3ad62dfb1f6141a0bf90bca6aa25508d
21:03:39.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.505267 32036 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:03:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.505111 32033 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "24794a00822242cba0e345d7ec9fd95f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.509652 32033 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [sys.catalog]: This master's current role is: LEADER
21:03:39.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.533865 32036 catalog_manager.cc:1380] Generated new certificate authority record
21:03:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.536504 32036 catalog_manager.cc:1514] Loading token signing keys...
21:03:39.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.538196 31917 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } attempt: 5
21:03:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.555833 31917 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5: Bootstrap starting.
21:03:39.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.560351 31917 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5: Neither blocks nor log segments found. Creating new log.
21:03:39.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.563622 32036 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f: Generated new TSK 0
21:03:39.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.565287 31917 log.cc:826] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5: Log is configured to *not* fsync() on all Append() calls
21:03:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.567008 32036 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:03:39.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.578027 31917 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5: No bootstrap required, opened a new log
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.579978 31749 external_mini_cluster.cc:1366] Running /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/wal
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/data
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/logs
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/data/info.pb
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.65:0
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.1.65
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.65
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:03:39.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.586304 31917 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:39.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.586719 31917 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:39.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.586848 31917 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6bf55660ec6046509a1e498a52e393e5, State: Initialized, Role: FOLLOWER
21:03:39.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.587788 31917 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:39.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.589248 32045 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:39.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.589962 32045 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.602530 31917 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [sys.catalog]: configured and running, proceeding with master startup.
21:03:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.602597 31882 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24794a00822242cba0e345d7ec9fd95f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf55660ec6046509a1e498a52e393e5" is_pre_election: true
21:03:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.603127 31882 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24794a00822242cba0e345d7ec9fd95f in term 0.
21:03:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.605221 31880 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 0 FOLLOWER]: Advancing to term 1
21:03:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.604482 31881 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24794a00822242cba0e345d7ec9fd95f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6bf55660ec6046509a1e498a52e393e5"
21:03:39.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.611129 31880 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 1 FOLLOWER]: Refusing update from remote peer 24794a00822242cba0e345d7ec9fd95f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:03:39.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.612774 32042 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:39.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.651844 32045 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24794a00822242cba0e345d7ec9fd95f. Latest consensus state: current_term: 1 leader_uuid: "24794a00822242cba0e345d7ec9fd95f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:39.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.652120 32045 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.652611 32053 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "24794a00822242cba0e345d7ec9fd95f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.652971 32053 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.711948 32033 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "24794a00822242cba0e345d7ec9fd95f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:39.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.712184 32033 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24794a00822242cba0e345d7ec9fd95f [sys.catalog]: This master's current role is: LEADER
21:03:39.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.715214 32061 catalog_manager.cc:1269] Loaded cluster ID: 3ad62dfb1f6141a0bf90bca6aa25508d
21:03:39.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.715474 32061 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5: loading cluster ID for follower catalog manager: success
21:03:39.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.719236 32061 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5: acquiring CA information for follower catalog manager: success
21:03:39.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:39.721715 32061 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:03:40.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.089715 32044 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:40.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.090030 32044 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:40.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.090171 32044 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:40.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.098336 32044 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:40.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.098503 32044 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:03:40.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.098573 32044 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:03:40.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.098626 32044 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:03:40.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.098758 32044 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.1.65
21:03:40.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.108546 32044 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:40.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:40.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:40.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:40.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/data
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/wal
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.65:0
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/data/info.pb
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.65
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.1.65
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/logs
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:23 UTC on 24a791456cd2
21:03:40.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9516
21:03:40.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:40.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.110440 32044 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:40.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.112212 32044 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:40.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.121472 32067 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:40.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.122382 32068 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:40.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.122897 32070 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:40.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.123694 32044 server_base.cc:1047] running on GCE node
21:03:40.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.124279 32044 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:40.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.125294 32044 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:40.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.126544 32044 hybrid_clock.cc:648] HybridClock initialized: now 1765746220126477 us; error 40 us; skew 500 ppm
21:03:40.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.126796 32044 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:40.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.129134 32044 webserver.cc:492] Webserver started at http://127.31.1.65:34895/ using document root <none> and password file <none>
21:03:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.129844 32044 fs_manager.cc:362] Metadata directory not provided
21:03:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.129980 32044 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.130293 32044 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:03:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.132786 32044 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/data/instance:
21:03:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34cad66201db44ecb5a55678bd60a0e6"
21:03:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:40 on dist-test-slave-2vjd"
21:03:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.133584 32044 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/wal/instance:
21:03:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34cad66201db44ecb5a55678bd60a0e6"
21:03:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:40 on dist-test-slave-2vjd"
21:03:40.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.138836 32044 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.004s sys 0.001s
21:03:40.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.142215 32076 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:40.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.143599 32044 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:03:40.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.143769 32044 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/data,/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/wal
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34cad66201db44ecb5a55678bd60a0e6"
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:40 on dist-test-slave-2vjd"
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.143961 32044 fs_report.cc:389] FS layout report
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/wal
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/wal
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/data/data
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:40.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:40.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.170315 32044 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:40.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.171243 32044 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:40.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.171555 32044 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:40.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.174808 32044 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:40.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.174928 32044 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:03:40.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.175034 32044 ts_tablet_manager.cc:616] Registered 0 tablets
21:03:40.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.175264 32044 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
21:03:40.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.265676 32044 rpc_server.cc:307] RPC server started. Bound to: 127.31.1.65:46755
21:03:40.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.265749 32200 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.1.65:46755 every 8 connection(s)
21:03:40.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.267000 32044 server_base.cc:1179] Dumped server information to /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/data/info.pb
21:03:40.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.277947 31749 external_mini_cluster.cc:1428] Started /tmp/dist-test-task6gQn2r/build/asan/bin/kudu as pid 32044
21:03:40.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.278193 31749 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-0/wal/instance
21:03:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.280279 32202 heartbeater.cc:344] Connected to a master server at 127.31.1.126:41315
21:03:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.281587 32202 heartbeater.cc:461] Registering TS with master...
21:03:40.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.282531 32202 heartbeater.cc:507] Master 127.31.1.126:41315 requested a full tablet report, sending...
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.283149 31749 external_mini_cluster.cc:1366] Running /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/wal
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/data
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/logs
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/data/info.pb
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.66:0
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.1.66
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.66
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:40.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:03:40.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.285321 31791 ts_manager.cc:194] Registered new tserver with Master: 34cad66201db44ecb5a55678bd60a0e6 (127.31.1.65:46755)
21:03:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.288008 32201 heartbeater.cc:344] Connected to a master server at 127.31.1.124:41201
21:03:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.288167 32201 heartbeater.cc:461] Registering TS with master...
21:03:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.288851 32201 heartbeater.cc:507] Master 127.31.1.124:41201 requested a full tablet report, sending...
21:03:40.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.290735 31953 ts_manager.cc:194] Registered new tserver with Master: 34cad66201db44ecb5a55678bd60a0e6 (127.31.1.65:46755)
21:03:40.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.293352 31953 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.31.1.65:54363
21:03:40.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.301857 32203 heartbeater.cc:344] Connected to a master server at 127.31.1.125:33977
21:03:40.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.302129 32203 heartbeater.cc:461] Registering TS with master...
21:03:40.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.302527 32203 heartbeater.cc:507] Master 127.31.1.125:33977 requested a full tablet report, sending...
21:03:40.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.303966 31872 ts_manager.cc:194] Registered new tserver with Master: 34cad66201db44ecb5a55678bd60a0e6 (127.31.1.65:46755)
21:03:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.400193 32032 catalog_manager.cc:1269] Loaded cluster ID: 3ad62dfb1f6141a0bf90bca6aa25508d
21:03:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.400303 32032 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c: loading cluster ID for follower catalog manager: success
21:03:40.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.403879 32032 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c: acquiring CA information for follower catalog manager: success
21:03:40.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.406145 32032 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:03:40.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.611172 32204 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:40.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.611461 32204 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:40.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.611550 32204 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.619392 32204 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.619513 32204 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:03:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.619560 32204 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:03:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.619593 32204 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:03:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.619674 32204 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.1.66
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.628767 32204 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/data
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/wal
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.66:0
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/data/info.pb
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.66
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.1.66
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/logs
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:23 UTC on 24a791456cd2
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9516
21:03:40.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:40.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.630532 32204 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:40.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.632159 32204 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:40.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.640663 32210 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:40.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.640658 32211 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:40.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:40.641474 32213 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:40.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.642874 32204 server_base.cc:1047] running on GCE node
21:03:40.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.643368 32204 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:40.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.644129 32204 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:40.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.645335 32204 hybrid_clock.cc:648] HybridClock initialized: now 1765746220645283 us; error 26 us; skew 500 ppm
21:03:40.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.645637 32204 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:40.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.647603 32204 webserver.cc:492] Webserver started at http://127.31.1.66:43921/ using document root <none> and password file <none>
21:03:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.648267 32204 fs_manager.cc:362] Metadata directory not provided
21:03:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.648384 32204 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.648741 32204 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:03:40.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.651073 32204 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/data/instance:
21:03:40.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc"
21:03:40.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:40 on dist-test-slave-2vjd"
21:03:40.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.651808 32204 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/wal/instance:
21:03:40.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc"
21:03:40.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:40 on dist-test-slave-2vjd"
21:03:40.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.656306 32204 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.000s sys 0.006s
21:03:40.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.659276 32219 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.660439 32204 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
21:03:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.660597 32204 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/data,/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/wal
21:03:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc"
21:03:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:40 on dist-test-slave-2vjd"
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.660828 32204 fs_report.cc:389] FS layout report
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/wal
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/wal
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/data/data
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:40.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.678236 32204 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.678993 32204 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.679267 32204 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.682149 32204 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.682300 32204 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:03:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.682406 32204 ts_tablet_manager.cc:616] Registered 0 tablets
21:03:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.682471 32204 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:03:40.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.734846 32204 rpc_server.cc:307] RPC server started. Bound to: 127.31.1.66:34399
21:03:40.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.734897 32340 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.1.66:34399 every 8 connection(s)
21:03:40.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.736029 32204 server_base.cc:1179] Dumped server information to /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/data/info.pb
21:03:40.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.739504 32342 heartbeater.cc:344] Connected to a master server at 127.31.1.126:41315
21:03:40.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.739888 32342 heartbeater.cc:461] Registering TS with master...
21:03:40.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.740696 32342 heartbeater.cc:507] Master 127.31.1.126:41315 requested a full tablet report, sending...
21:03:40.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.742461 31791 ts_manager.cc:194] Registered new tserver with Master: ba7e9fe3bc2f4c22917ab9ae27e91fdc (127.31.1.66:34399)
21:03:40.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.744427 32341 heartbeater.cc:344] Connected to a master server at 127.31.1.124:41201
21:03:40.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.744532 32341 heartbeater.cc:461] Registering TS with master...
21:03:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.744944 31749 external_mini_cluster.cc:1428] Started /tmp/dist-test-task6gQn2r/build/asan/bin/kudu as pid 32204
21:03:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.745031 32343 heartbeater.cc:344] Connected to a master server at 127.31.1.125:33977
21:03:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.745060 32341 heartbeater.cc:507] Master 127.31.1.124:41201 requested a full tablet report, sending...
21:03:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.745138 31749 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-1/wal/instance
21:03:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.745178 32343 heartbeater.cc:461] Registering TS with master...
21:03:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.745638 32343 heartbeater.cc:507] Master 127.31.1.125:33977 requested a full tablet report, sending...
21:03:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.745962 31953 ts_manager.cc:194] Registered new tserver with Master: ba7e9fe3bc2f4c22917ab9ae27e91fdc (127.31.1.66:34399)
21:03:40.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.746991 31872 ts_manager.cc:194] Registered new tserver with Master: ba7e9fe3bc2f4c22917ab9ae27e91fdc (127.31.1.66:34399)
21:03:40.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.747048 31953 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.31.1.66:46739
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:40.749318 31749 external_mini_cluster.cc:1366] Running /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task6gQn2r/build/asan/bin/kudu
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/wal
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/data
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/logs
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/data/info.pb
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.67:0
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.1.67
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.67
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:40.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:03:41.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.064138 32346 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:03:41.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.064442 32346 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:41.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.064579 32346 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:03:41.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.072340 32346 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:03:41.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.072501 32346 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:03:41.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.072571 32346 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:03:41.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.072621 32346 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:03:41.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.072751 32346 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.1.67
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.081910 32346 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.1.84:35119
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/data
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/wal
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.1.67:0
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/data/info.pb
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.1.67
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.1.67
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/logs
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:23 UTC on 24a791456cd2
21:03:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9516
21:03:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:03:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.083642 32346 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:41.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.085340 32346 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:03:41.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.094264 32351 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:41.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.094182 32352 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:41.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.094231 32354 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:41.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.095510 32346 server_base.cc:1047] running on GCE node
21:03:41.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.096051 32346 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:41.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.096908 32346 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:41.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.098125 32346 hybrid_clock.cc:648] HybridClock initialized: now 1765746221098062 us; error 35 us; skew 500 ppm
21:03:41.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.098402 32346 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.100438 32346 webserver.cc:492] Webserver started at http://127.31.1.67:37089/ using document root <none> and password file <none>
21:03:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.101179 32346 fs_manager.cc:362] Metadata directory not provided
21:03:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.101315 32346 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.101622 32346 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:03:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.104065 32346 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/data/instance:
21:03:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "66725a0404584859829bf57f228209a4"
21:03:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:41 on dist-test-slave-2vjd"
21:03:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.105090 32346 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/wal/instance:
21:03:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "66725a0404584859829bf57f228209a4"
21:03:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:41 on dist-test-slave-2vjd"
21:03:41.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.110045 32346 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.000s sys 0.004s
21:03:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.113441 32360 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.114758 32346 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.114981 32346 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/data,/tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/wal
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "66725a0404584859829bf57f228209a4"
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:03:41 on dist-test-slave-2vjd"
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.115159 32346 fs_report.cc:389] FS layout report
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/wal
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/wal
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/data/data
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:41.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.139854 32346 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:41.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.140791 32346 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:41.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.141110 32346 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.144064 32346 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.144184 32346 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:03:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.144292 32346 ts_tablet_manager.cc:616] Registered 0 tablets
21:03:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.144394 32346 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:03:41.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.195015 32346 rpc_server.cc:307] RPC server started. Bound to: 127.31.1.67:36481
21:03:41.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.195052 32481 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.1.67:36481 every 8 connection(s)
21:03:41.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.196254 32346 server_base.cc:1179] Dumped server information to /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/data/info.pb
21:03:41.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.200044 32483 heartbeater.cc:344] Connected to a master server at 127.31.1.126:41315
21:03:41.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.200424 32483 heartbeater.cc:461] Registering TS with master...
21:03:41.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.201503 32483 heartbeater.cc:507] Master 127.31.1.126:41315 requested a full tablet report, sending...
21:03:41.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.203341 31791 ts_manager.cc:194] Registered new tserver with Master: 66725a0404584859829bf57f228209a4 (127.31.1.67:36481)
21:03:41.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.205061 31749 external_mini_cluster.cc:1428] Started /tmp/dist-test-task6gQn2r/build/asan/bin/kudu as pid 32346
21:03:41.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.205224 31749 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task6gQn2r/test-tmp/mini-kudu-cluster8485440040081086843/ts-2/wal/instance
21:03:41.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.206857 32482 heartbeater.cc:344] Connected to a master server at 127.31.1.124:41201
21:03:41.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.206883 32484 heartbeater.cc:344] Connected to a master server at 127.31.1.125:33977
21:03:41.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.207136 32484 heartbeater.cc:461] Registering TS with master...
21:03:41.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.207188 32482 heartbeater.cc:461] Registering TS with master...
21:03:41.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.207762 32484 heartbeater.cc:507] Master 127.31.1.125:33977 requested a full tablet report, sending...
21:03:41.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.207844 32482 heartbeater.cc:507] Master 127.31.1.124:41201 requested a full tablet report, sending...
21:03:41.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.208891 31872 ts_manager.cc:194] Registered new tserver with Master: 66725a0404584859829bf57f228209a4 (127.31.1.67:36481)
21:03:41.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.208902 31952 ts_manager.cc:194] Registered new tserver with Master: 66725a0404584859829bf57f228209a4 (127.31.1.67:36481)
21:03:41.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.209995 31952 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.31.1.67:40721
21:03:41.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.213743 31749 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:03:41.215 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:03:41.215 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:03:41.216 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.31.1.126"
port: 41315
}
bound_http_address {
host: "127.31.1.126"
port: 46507
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.31.1.125"
port: 33977
}
bound_http_address {
host: "127.31.1.125"
port: 44409
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.31.1.124"
port: 41201
}
bound_http_address {
host: "127.31.1.124"
port: 42021
}
}
}
21:03:41.216 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:03:41.217 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.31.1.65"
port: 46755
}
bound_http_address {
host: "127.31.1.65"
port: 34895
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.31.1.66"
port: 34399
}
bound_http_address {
host: "127.31.1.66"
port: 43921
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.31.1.67"
port: 36481
}
bound_http_address {
host: "127.31.1.67"
port: 37089
}
}
}
21:03:41.217 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:03:41.218 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:03:41.220 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.31.1.126:41315(127.31.1.126:41315)] connecting to peer
21:03:41.221 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.31.1.125:33977(127.31.1.125:33977)] connecting to peer
21:03:41.222 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:41.222 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.31.1.126:41315(127.31.1.126:41315)] Successfully connected to peer
21:03:41.223 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.31.1.125:33977(127.31.1.125:33977)] Successfully connected to peer
21:03:41.224 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.31.1.126
21:03:41.227 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.31.1.124:41201(127.31.1.124:41201)] Successfully connected to peer
21:03:41.228 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.31.1.124
21:03:41.228 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.31.1.125
21:03:41.239 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x97805807, L:/127.0.0.1:34378 - R:/127.31.1.125:33977] using SASL/PLAIN
21:03:41.241 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4c578fbc, L:/127.0.0.1:50216 - R:/127.31.1.126:41315] using SASL/PLAIN
21:03:41.242 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xeb095235, L:/127.0.0.1:57126 - R:/127.31.1.124:41201] using SASL/PLAIN
21:03:41.244 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:03:41.244 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=25), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [21ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: OK, deferred=null) after lookup
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.246270 31953 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57126:
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:41.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.248101 31953 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:03:41.262 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 4b1bd3ea937a4e26ae7fdf112e5fadaa
21:03:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.302228 32201 heartbeater.cc:499] Master 127.31.1.124:41201 was elected leader, sending a full tablet report...
21:03:41.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.315122 32275 tablet_service.cc:1505] Processing CreateTablet for tablet e9c747530e3a4b36864f2ab09e4d7436 (DEFAULT_TABLE table=txn_manager_fallback [id=4b1bd3ea937a4e26ae7fdf112e5fadaa]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:03:41.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.315389 32130 tablet_service.cc:1505] Processing CreateTablet for tablet aa55302b73564eb887f0a64a87542411 (DEFAULT_TABLE table=txn_manager_fallback [id=4b1bd3ea937a4e26ae7fdf112e5fadaa]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:03:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.315990 32276 tablet_service.cc:1505] Processing CreateTablet for tablet aa55302b73564eb887f0a64a87542411 (DEFAULT_TABLE table=txn_manager_fallback [id=4b1bd3ea937a4e26ae7fdf112e5fadaa]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:03:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.315609 32129 tablet_service.cc:1505] Processing CreateTablet for tablet e9c747530e3a4b36864f2ab09e4d7436 (DEFAULT_TABLE table=txn_manager_fallback [id=4b1bd3ea937a4e26ae7fdf112e5fadaa]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:03:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.316794 32130 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aa55302b73564eb887f0a64a87542411. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.316821 32275 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e9c747530e3a4b36864f2ab09e4d7436. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:41.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.317226 32129 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e9c747530e3a4b36864f2ab09e4d7436. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:41.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.318406 32276 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aa55302b73564eb887f0a64a87542411. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:41.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.321080 32410 tablet_service.cc:1505] Processing CreateTablet for tablet aa55302b73564eb887f0a64a87542411 (DEFAULT_TABLE table=txn_manager_fallback [id=4b1bd3ea937a4e26ae7fdf112e5fadaa]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:03:41.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.321080 32409 tablet_service.cc:1505] Processing CreateTablet for tablet e9c747530e3a4b36864f2ab09e4d7436 (DEFAULT_TABLE table=txn_manager_fallback [id=4b1bd3ea937a4e26ae7fdf112e5fadaa]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:03:41.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.322654 32410 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aa55302b73564eb887f0a64a87542411. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:41.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.323439 32409 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e9c747530e3a4b36864f2ab09e4d7436. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.335273 32507 tablet_bootstrap.cc:492] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6: Bootstrap starting.
21:03:41.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.338692 32509 tablet_bootstrap.cc:492] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4: Bootstrap starting.
21:03:41.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.339488 32508 tablet_bootstrap.cc:492] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Bootstrap starting.
21:03:41.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.339777 32507 tablet_bootstrap.cc:654] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6: Neither blocks nor log segments found. Creating new log.
21:03:41.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.341087 32507 log.cc:826] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6: Log is configured to *not* fsync() on all Append() calls
21:03:41.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.342370 32509 tablet_bootstrap.cc:654] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4: Neither blocks nor log segments found. Creating new log.
21:03:41.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.342967 32508 tablet_bootstrap.cc:654] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Neither blocks nor log segments found. Creating new log.
21:03:41.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.343756 32509 log.cc:826] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4: Log is configured to *not* fsync() on all Append() calls
21:03:41.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.344187 32508 log.cc:826] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Log is configured to *not* fsync() on all Append() calls
21:03:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.346750 32509 tablet_bootstrap.cc:492] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4: No bootstrap required, opened a new log
21:03:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.346881 32508 tablet_bootstrap.cc:492] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: No bootstrap required, opened a new log
21:03:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.347096 32509 ts_tablet_manager.cc:1403] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4: Time spent bootstrapping tablet: real 0.009s user 0.008s sys 0.000s
21:03:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.347096 32508 ts_tablet_manager.cc:1403] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.000s
21:03:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.347977 32507 tablet_bootstrap.cc:492] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6: No bootstrap required, opened a new log
21:03:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.348217 32507 ts_tablet_manager.cc:1403] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6: Time spent bootstrapping tablet: real 0.013s user 0.010s sys 0.000s
21:03:41.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.352909 32507 raft_consensus.cc:359] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.353212 32507 raft_consensus.cc:385] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:41.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.353330 32507 raft_consensus.cc:740] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 34cad66201db44ecb5a55678bd60a0e6, State: Initialized, Role: FOLLOWER
21:03:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.353921 32507 consensus_queue.cc:260] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.354103 32509 raft_consensus.cc:359] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.354104 32508 raft_consensus.cc:359] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.354462 32508 raft_consensus.cc:385] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.354473 32509 raft_consensus.cc:385] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.354561 32508 raft_consensus.cc:740] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba7e9fe3bc2f4c22917ab9ae27e91fdc, State: Initialized, Role: FOLLOWER
21:03:41.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.354561 32509 raft_consensus.cc:740] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 66725a0404584859829bf57f228209a4, State: Initialized, Role: FOLLOWER
21:03:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.355254 32508 consensus_queue.cc:260] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.355234 32509 consensus_queue.cc:260] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.355810 32507 ts_tablet_manager.cc:1434] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6: Time spent starting tablet: real 0.007s user 0.004s sys 0.004s
21:03:41.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.356242 32507 tablet_bootstrap.cc:492] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6: Bootstrap starting.
21:03:41.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.357977 32507 tablet_bootstrap.cc:654] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6: Neither blocks nor log segments found. Creating new log.
21:03:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.366513 32482 heartbeater.cc:499] Master 127.31.1.124:41201 was elected leader, sending a full tablet report...
21:03:41.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.366955 32507 tablet_bootstrap.cc:492] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6: No bootstrap required, opened a new log
21:03:41.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.367136 32507 ts_tablet_manager.cc:1403] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6: Time spent bootstrapping tablet: real 0.011s user 0.008s sys 0.000s
21:03:41.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.367710 32509 ts_tablet_manager.cc:1434] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4: Time spent starting tablet: real 0.021s user 0.008s sys 0.007s
21:03:41.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.367933 32507 raft_consensus.cc:359] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.368131 32507 raft_consensus.cc:385] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:41.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.368232 32507 raft_consensus.cc:740] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 34cad66201db44ecb5a55678bd60a0e6, State: Initialized, Role: FOLLOWER
21:03:41.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.368502 32507 consensus_queue.cc:260] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.371533 32507 ts_tablet_manager.cc:1434] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
21:03:41.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.375037 32509 tablet_bootstrap.cc:492] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4: Bootstrap starting.
21:03:41.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.376191 32341 heartbeater.cc:499] Master 127.31.1.124:41201 was elected leader, sending a full tablet report...
21:03:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.376920 32509 tablet_bootstrap.cc:654] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4: Neither blocks nor log segments found. Creating new log.
21:03:41.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.376901 32508 ts_tablet_manager.cc:1434] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Time spent starting tablet: real 0.030s user 0.016s sys 0.008s
21:03:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.380383 32508 tablet_bootstrap.cc:492] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Bootstrap starting.
21:03:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.382273 32509 tablet_bootstrap.cc:492] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4: No bootstrap required, opened a new log
21:03:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.382421 32508 tablet_bootstrap.cc:654] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Neither blocks nor log segments found. Creating new log.
21:03:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.382448 32509 ts_tablet_manager.cc:1403] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.006s
21:03:41.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.383247 32509 raft_consensus.cc:359] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.383422 32509 raft_consensus.cc:385] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:41.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.383513 32509 raft_consensus.cc:740] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 66725a0404584859829bf57f228209a4, State: Initialized, Role: FOLLOWER
21:03:41.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.383798 32509 consensus_queue.cc:260] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.384339 32509 ts_tablet_manager.cc:1434] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
21:03:41.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.384497 32508 tablet_bootstrap.cc:492] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: No bootstrap required, opened a new log
21:03:41.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.384630 32508 ts_tablet_manager.cc:1403] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.000s
21:03:41.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.385439 32508 raft_consensus.cc:359] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.385619 32508 raft_consensus.cc:385] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:41.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.385727 32508 raft_consensus.cc:740] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba7e9fe3bc2f4c22917ab9ae27e91fdc, State: Initialized, Role: FOLLOWER
21:03:41.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.385950 32508 consensus_queue.cc:260] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.386511 32508 ts_tablet_manager.cc:1434] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
21:03:41.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.449023 32485 tablet.cc:2378] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.488765 32344 tablet.cc:2378] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:41.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.519863 32516 raft_consensus.cc:493] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:41.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.520211 32516 raft_consensus.cc:515] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.522006 32516 leader_election.cc:290] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34cad66201db44ecb5a55678bd60a0e6 (127.31.1.65:46755), 66725a0404584859829bf57f228209a4 (127.31.1.67:36481)
21:03:41.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.529074 32434 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "e9c747530e3a4b36864f2ab09e4d7436" candidate_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "66725a0404584859829bf57f228209a4" is_pre_election: true
21:03:41.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.529419 32151 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "e9c747530e3a4b36864f2ab09e4d7436" candidate_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "34cad66201db44ecb5a55678bd60a0e6" is_pre_election: true
21:03:41.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.529536 32434 raft_consensus.cc:2468] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba7e9fe3bc2f4c22917ab9ae27e91fdc in term 0.
21:03:41.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.529884 32151 raft_consensus.cc:2468] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba7e9fe3bc2f4c22917ab9ae27e91fdc in term 0.
21:03:41.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.530153 32220 leader_election.cc:304] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 66725a0404584859829bf57f228209a4, ba7e9fe3bc2f4c22917ab9ae27e91fdc; no voters:
21:03:41.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.530536 32516 raft_consensus.cc:2804] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Leader pre-election won for term 1
21:03:41.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.530714 32516 raft_consensus.cc:493] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:41.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.530809 32516 raft_consensus.cc:3060] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Advancing to term 1
21:03:41.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.532799 32516 raft_consensus.cc:515] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.533330 32516 leader_election.cc:290] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [CANDIDATE]: Term 1 election: Requested vote from peers 34cad66201db44ecb5a55678bd60a0e6 (127.31.1.65:46755), 66725a0404584859829bf57f228209a4 (127.31.1.67:36481)
21:03:41.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.533763 32151 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "e9c747530e3a4b36864f2ab09e4d7436" candidate_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "34cad66201db44ecb5a55678bd60a0e6"
21:03:41.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.533870 32434 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "e9c747530e3a4b36864f2ab09e4d7436" candidate_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "66725a0404584859829bf57f228209a4"
21:03:41.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.533977 32151 raft_consensus.cc:3060] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Advancing to term 1
21:03:41.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.534055 32434 raft_consensus.cc:3060] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Advancing to term 1
21:03:41.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.536934 32151 raft_consensus.cc:2468] T e9c747530e3a4b36864f2ab09e4d7436 P 34cad66201db44ecb5a55678bd60a0e6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba7e9fe3bc2f4c22917ab9ae27e91fdc in term 1.
21:03:41.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.536934 32434 raft_consensus.cc:2468] T e9c747530e3a4b36864f2ab09e4d7436 P 66725a0404584859829bf57f228209a4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba7e9fe3bc2f4c22917ab9ae27e91fdc in term 1.
21:03:41.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.537350 32220 leader_election.cc:304] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 34cad66201db44ecb5a55678bd60a0e6, 66725a0404584859829bf57f228209a4, ba7e9fe3bc2f4c22917ab9ae27e91fdc; no voters:
21:03:41.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.537665 32516 raft_consensus.cc:2804] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 1 FOLLOWER]: Leader election won for term 1
21:03:41.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.538095 32516 raft_consensus.cc:697] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 1 LEADER]: Becoming Leader. State: Replica: ba7e9fe3bc2f4c22917ab9ae27e91fdc, State: Running, Role: LEADER
21:03:41.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.538601 32516 consensus_queue.cc:237] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.543984 31952 catalog_manager.cc:5654] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc reported cstate change: term changed from 0 to 1, leader changed from <none> to ba7e9fe3bc2f4c22917ab9ae27e91fdc (127.31.1.66). New cstate: current_term: 1 leader_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } health_report { overall_health: UNKNOWN } } }
21:03:41.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:41.553242 32205 tablet.cc:2378] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:41.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.677453 32513 raft_consensus.cc:493] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:41.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.677662 32513 raft_consensus.cc:515] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.679280 32513 leader_election.cc:290] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ba7e9fe3bc2f4c22917ab9ae27e91fdc (127.31.1.66:34399), 66725a0404584859829bf57f228209a4 (127.31.1.67:36481)
21:03:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.687577 32296 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "aa55302b73564eb887f0a64a87542411" candidate_uuid: "34cad66201db44ecb5a55678bd60a0e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" is_pre_election: true
21:03:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.688009 32296 raft_consensus.cc:2468] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 34cad66201db44ecb5a55678bd60a0e6 in term 0.
21:03:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.687947 32434 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "aa55302b73564eb887f0a64a87542411" candidate_uuid: "34cad66201db44ecb5a55678bd60a0e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "66725a0404584859829bf57f228209a4" is_pre_election: true
21:03:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.688190 32434 raft_consensus.cc:2468] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 34cad66201db44ecb5a55678bd60a0e6 in term 0.
21:03:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.688560 32077 leader_election.cc:304] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34cad66201db44ecb5a55678bd60a0e6, ba7e9fe3bc2f4c22917ab9ae27e91fdc; no voters:
21:03:41.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.688953 32513 raft_consensus.cc:2804] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:03:41.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.689081 32513 raft_consensus.cc:493] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:41.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.689186 32513 raft_consensus.cc:3060] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Advancing to term 1
21:03:41.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.691116 32513 raft_consensus.cc:515] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.691679 32513 leader_election.cc:290] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [CANDIDATE]: Term 1 election: Requested vote from peers ba7e9fe3bc2f4c22917ab9ae27e91fdc (127.31.1.66:34399), 66725a0404584859829bf57f228209a4 (127.31.1.67:36481)
21:03:41.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.692058 32296 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "aa55302b73564eb887f0a64a87542411" candidate_uuid: "34cad66201db44ecb5a55678bd60a0e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc"
21:03:41.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.692216 32296 raft_consensus.cc:3060] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Advancing to term 1
21:03:41.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.692150 32434 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "aa55302b73564eb887f0a64a87542411" candidate_uuid: "34cad66201db44ecb5a55678bd60a0e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "66725a0404584859829bf57f228209a4"
21:03:41.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.692399 32434 raft_consensus.cc:3060] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Advancing to term 1
21:03:41.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.694227 32296 raft_consensus.cc:2468] T aa55302b73564eb887f0a64a87542411 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 34cad66201db44ecb5a55678bd60a0e6 in term 1.
21:03:41.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.694248 32434 raft_consensus.cc:2468] T aa55302b73564eb887f0a64a87542411 P 66725a0404584859829bf57f228209a4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 34cad66201db44ecb5a55678bd60a0e6 in term 1.
21:03:41.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.694614 32077 leader_election.cc:304] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34cad66201db44ecb5a55678bd60a0e6, 66725a0404584859829bf57f228209a4; no voters:
21:03:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.694917 32513 raft_consensus.cc:2804] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [term 1 FOLLOWER]: Leader election won for term 1
21:03:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.695348 32513 raft_consensus.cc:697] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [term 1 LEADER]: Becoming Leader. State: Replica: 34cad66201db44ecb5a55678bd60a0e6, State: Running, Role: LEADER
21:03:41.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.695947 32513 consensus_queue.cc:237] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:41.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.700960 31952 catalog_manager.cc:5654] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 34cad66201db44ecb5a55678bd60a0e6 (127.31.1.65). New cstate: current_term: 1 leader_uuid: "34cad66201db44ecb5a55678bd60a0e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } health_report { overall_health: UNKNOWN } } }
21:03:41.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.977227 32516 consensus_queue.cc:1048] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [LEADER]: Connected to new peer: Peer: permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:41.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:41.988588 32521 consensus_queue.cc:1048] T e9c747530e3a4b36864f2ab09e4d7436 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [LEADER]: Connected to new peer: Peer: permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.155193 32513 consensus_queue.cc:1048] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:42.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.160786 32526 consensus_queue.cc:1048] T aa55302b73564eb887f0a64a87542411 P 34cad66201db44ecb5a55678bd60a0e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.488735 31953 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57138:
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:03:42.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:42.489683 31953 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:03:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.503396 32130 tablet_service.cc:1505] Processing CreateTablet for tablet bff3b2b38a50457497ad56d0ced46e75 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=fdcb6236b99b439ba664548dbfc6b601]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:03:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.504415 32130 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bff3b2b38a50457497ad56d0ced46e75. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.504235 32275 tablet_service.cc:1505] Processing CreateTablet for tablet bff3b2b38a50457497ad56d0ced46e75 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=fdcb6236b99b439ba664548dbfc6b601]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:03:42.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.504370 32409 tablet_service.cc:1505] Processing CreateTablet for tablet bff3b2b38a50457497ad56d0ced46e75 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=fdcb6236b99b439ba664548dbfc6b601]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:03:42.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.505168 32275 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bff3b2b38a50457497ad56d0ced46e75. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:42.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.505471 32409 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bff3b2b38a50457497ad56d0ced46e75. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:42.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.510447 32555 tablet_bootstrap.cc:492] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Bootstrap starting.
21:03:42.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.511987 32555 tablet_bootstrap.cc:654] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Neither blocks nor log segments found. Creating new log.
21:03:42.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.518718 32556 tablet_bootstrap.cc:492] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: Bootstrap starting.
21:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.521445 32558 tablet_bootstrap.cc:492] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4: Bootstrap starting.
21:03:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.521849 32556 tablet_bootstrap.cc:654] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: Neither blocks nor log segments found. Creating new log.
21:03:42.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.523003 32558 tablet_bootstrap.cc:654] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4: Neither blocks nor log segments found. Creating new log.
21:03:42.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.523581 32556 tablet_bootstrap.cc:492] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: No bootstrap required, opened a new log
21:03:42.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.523695 32556 ts_tablet_manager.cc:1403] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
21:03:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.524191 32555 tablet_bootstrap.cc:492] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: No bootstrap required, opened a new log
21:03:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.524339 32555 ts_tablet_manager.cc:1403] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Time spent bootstrapping tablet: real 0.014s user 0.000s sys 0.010s
21:03:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.524518 32556 raft_consensus.cc:359] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.524662 32556 raft_consensus.cc:385] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.524760 32556 raft_consensus.cc:740] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 34cad66201db44ecb5a55678bd60a0e6, State: Initialized, Role: FOLLOWER
21:03:42.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.524994 32556 consensus_queue.cc:260] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:42.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.525220 32555 raft_consensus.cc:359] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:42.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.525388 32526 tablet_replica.cc:442] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } } }
21:03:42.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.525494 32555 raft_consensus.cc:385] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:42.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.525653 32526 tablet_replica.cc:445] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:42.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.525703 32558 tablet_bootstrap.cc:492] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4: No bootstrap required, opened a new log
21:03:42.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.525825 32558 ts_tablet_manager.cc:1403] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
21:03:42.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.526146 32555 raft_consensus.cc:740] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba7e9fe3bc2f4c22917ab9ae27e91fdc, State: Initialized, Role: FOLLOWER
21:03:42.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.526476 32555 consensus_queue.cc:260] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.526911 32555 ts_tablet_manager.cc:1434] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
21:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.527108 32516 tablet_replica.cc:442] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } } }
21:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.527364 32516 tablet_replica.cc:445] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:42.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.528760 32556 ts_tablet_manager.cc:1434] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
21:03:42.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.529378 32558 raft_consensus.cc:359] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:42.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.531730 32558 raft_consensus.cc:385] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:42.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.531836 32558 raft_consensus.cc:740] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 66725a0404584859829bf57f228209a4, State: Initialized, Role: FOLLOWER
21:03:42.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.532114 32558 consensus_queue.cc:260] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:42.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.532593 32558 ts_tablet_manager.cc:1434] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4: Time spent starting tablet: real 0.006s user 0.001s sys 0.000s
21:03:42.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.532516 32528 tablet_replica.cc:442] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } } }
21:03:42.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.532800 32528 tablet_replica.cc:445] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:42.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.677078 32526 raft_consensus.cc:493] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:42.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.677289 32526 raft_consensus.cc:515] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:42.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.677923 32526 leader_election.cc:290] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ba7e9fe3bc2f4c22917ab9ae27e91fdc (127.31.1.66:34399), 66725a0404584859829bf57f228209a4 (127.31.1.67:36481)
21:03:42.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.678530 32296 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "bff3b2b38a50457497ad56d0ced46e75" candidate_uuid: "34cad66201db44ecb5a55678bd60a0e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" is_pre_election: true
21:03:42.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.678637 32434 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "bff3b2b38a50457497ad56d0ced46e75" candidate_uuid: "34cad66201db44ecb5a55678bd60a0e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "66725a0404584859829bf57f228209a4" is_pre_election: true
21:03:42.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.678833 32296 raft_consensus.cc:2468] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 34cad66201db44ecb5a55678bd60a0e6 in term 0.
21:03:42.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.678853 32434 raft_consensus.cc:2468] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 34cad66201db44ecb5a55678bd60a0e6 in term 0.
21:03:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.679276 32077 leader_election.cc:304] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34cad66201db44ecb5a55678bd60a0e6, ba7e9fe3bc2f4c22917ab9ae27e91fdc; no voters:
21:03:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.679564 32526 raft_consensus.cc:2804] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:03:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.679651 32526 raft_consensus.cc:493] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.679723 32526 raft_consensus.cc:3060] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [term 0 FOLLOWER]: Advancing to term 1
21:03:42.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.681496 32526 raft_consensus.cc:515] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.682071 32526 leader_election.cc:290] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [CANDIDATE]: Term 1 election: Requested vote from peers ba7e9fe3bc2f4c22917ab9ae27e91fdc (127.31.1.66:34399), 66725a0404584859829bf57f228209a4 (127.31.1.67:36481)
21:03:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.682526 32296 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "bff3b2b38a50457497ad56d0ced46e75" candidate_uuid: "34cad66201db44ecb5a55678bd60a0e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc"
21:03:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.682598 32434 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "bff3b2b38a50457497ad56d0ced46e75" candidate_uuid: "34cad66201db44ecb5a55678bd60a0e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "66725a0404584859829bf57f228209a4"
21:03:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.682730 32296 raft_consensus.cc:3060] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 0 FOLLOWER]: Advancing to term 1
21:03:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.682780 32434 raft_consensus.cc:3060] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4 [term 0 FOLLOWER]: Advancing to term 1
21:03:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.684829 32434 raft_consensus.cc:2468] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 34cad66201db44ecb5a55678bd60a0e6 in term 1.
21:03:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.684870 32296 raft_consensus.cc:2468] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 34cad66201db44ecb5a55678bd60a0e6 in term 1.
21:03:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.685236 32077 leader_election.cc:304] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34cad66201db44ecb5a55678bd60a0e6, 66725a0404584859829bf57f228209a4; no voters:
21:03:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.685564 32526 raft_consensus.cc:2804] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [term 1 FOLLOWER]: Leader election won for term 1
21:03:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.685940 32526 raft_consensus.cc:697] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [term 1 LEADER]: Becoming Leader. State: Replica: 34cad66201db44ecb5a55678bd60a0e6, State: Running, Role: LEADER
21:03:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.686236 32526 consensus_queue.cc:237] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } }
21:03:42.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.687728 32561 tablet_replica.cc:442] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: TxnStatusTablet state changed. Reason: New leader 34cad66201db44ecb5a55678bd60a0e6. Latest consensus state: current_term: 1 leader_uuid: "34cad66201db44ecb5a55678bd60a0e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } } }
21:03:42.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.688059 32561 tablet_replica.cc:445] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: This TxnStatusTablet replica's current role is: LEADER
21:03:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.688741 32563 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
21:03:42.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.689025 32563 txn_status_manager.cc:930] Loading transaction status metadata into memory...
21:03:42.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.689903 31952 catalog_manager.cc:5654] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 34cad66201db44ecb5a55678bd60a0e6 (127.31.1.65). New cstate: current_term: 1 leader_uuid: "34cad66201db44ecb5a55678bd60a0e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } health_report { overall_health: UNKNOWN } } }
21:03:42.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.719622 32434 raft_consensus.cc:1275] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4 [term 1 FOLLOWER]: Refusing update from remote peer 34cad66201db44ecb5a55678bd60a0e6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:03:42.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.719766 32296 raft_consensus.cc:1275] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc [term 1 FOLLOWER]: Refusing update from remote peer 34cad66201db44ecb5a55678bd60a0e6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:03:42.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.720278 32561 consensus_queue.cc:1048] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:42.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.720523 32526 consensus_queue.cc:1048] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:42.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.737035 32528 tablet_replica.cc:442] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4: TxnStatusTablet state changed. Reason: New leader 34cad66201db44ecb5a55678bd60a0e6. Latest consensus state: current_term: 1 leader_uuid: "34cad66201db44ecb5a55678bd60a0e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } } }
21:03:42.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.737283 32528 tablet_replica.cc:445] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:42.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.739929 32568 mvcc.cc:204] Tried to move back new op lower bound from 7232496528252538880 to 7232496528124940288. Current Snapshot: MvccSnapshot[applied={T|T < 7232496528252538880}]
21:03:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.742414 32526 tablet_replica.cc:442] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "34cad66201db44ecb5a55678bd60a0e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } } }
21:03:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.742647 32526 tablet_replica.cc:445] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: This TxnStatusTablet replica's current role is: LEADER
21:03:42.748 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
21:03:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.750456 32516 tablet_replica.cc:442] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: TxnStatusTablet state changed. Reason: New leader 34cad66201db44ecb5a55678bd60a0e6. Latest consensus state: current_term: 1 leader_uuid: "34cad66201db44ecb5a55678bd60a0e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } } }
21:03:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.750713 32516 tablet_replica.cc:445] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.751435 32561 tablet_replica.cc:442] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "34cad66201db44ecb5a55678bd60a0e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } } }
21:03:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.751647 32561 tablet_replica.cc:445] T bff3b2b38a50457497ad56d0ced46e75 P 34cad66201db44ecb5a55678bd60a0e6: This TxnStatusTablet replica's current role is: LEADER
21:03:42.755 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 4b1bd3ea937a4e26ae7fdf112e5fadaa
21:03:42.767 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet aa55302b73564eb887f0a64a87542411 for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
21:03:42.768 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet e9c747530e3a4b36864f2ab09e4d7436 for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
21:03:42.768 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=aa55302b73564eb887f0a64a87542411@[34cad66201db44ecb5a55678bd60a0e6(127.31.1.65:46755)[L],66725a0404584859829bf57f228209a4(127.31.1.67:36481),ba7e9fe3bc2f4c22917ab9ae27e91fdc(127.31.1.66:34399)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=e9c747530e3a4b36864f2ab09e4d7436@[34cad66201db44ecb5a55678bd60a0e6(127.31.1.65:46755),66725a0404584859829bf57f228209a4(127.31.1.67:36481),ba7e9fe3bc2f4c22917ab9ae27e91fdc(127.31.1.66:34399)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
21:03:42.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.767357 32528 tablet_replica.cc:442] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "34cad66201db44ecb5a55678bd60a0e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } } }
21:03:42.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.767592 32528 tablet_replica.cc:445] T bff3b2b38a50457497ad56d0ced46e75 P 66725a0404584859829bf57f228209a4: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:42.768 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.31.1.124:41201, [12ms] Sub RPC GetTableLocations: received response from server master-127.31.1.124:41201: OK, deferred=null) after lookup
21:03:42.770 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 34cad66201db44ecb5a55678bd60a0e6(127.31.1.65:46755)] connecting to peer
21:03:42.771 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 34cad66201db44ecb5a55678bd60a0e6(127.31.1.65:46755)] Successfully connected to peer
21:03:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.772902 32516 tablet_replica.cc:442] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "34cad66201db44ecb5a55678bd60a0e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34cad66201db44ecb5a55678bd60a0e6" member_type: VOTER last_known_addr { host: "127.31.1.65" port: 46755 } } peers { permanent_uuid: "ba7e9fe3bc2f4c22917ab9ae27e91fdc" member_type: VOTER last_known_addr { host: "127.31.1.66" port: 34399 } } peers { permanent_uuid: "66725a0404584859829bf57f228209a4" member_type: VOTER last_known_addr { host: "127.31.1.67" port: 36481 } } }
21:03:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.773128 32516 tablet_replica.cc:445] T bff3b2b38a50457497ad56d0ced46e75 P ba7e9fe3bc2f4c22917ab9ae27e91fdc: This TxnStatusTablet replica's current role is: FOLLOWER
21:03:42.784 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x5f6dbd1c, L:/127.0.0.1:33426 - R:/127.31.1.65:46755] using TOKEN/null
21:03:42.822 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.31.1.124:41201
21:03:42.822 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:03:42.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.823347 31749 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task6gQn2r/build/asan/bin/kudu pid 31919
21:03:42.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.824906 31993 generic_service.cc:196] Checking for leaks (request via RPC)
21:03:42.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:42.955603 31749 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task6gQn2r/build/asan/bin/kudu with pid 31919
21:03:42.977 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.31.1.124:41201(127.31.1.124:41201)] handling channelInactive
21:03:42.977 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state READY due to: connection closed
21:03:42.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:42.977663 31856 connection.cc:537] server connection from 127.0.0.1:34370 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:03:42.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:42.978339 31776 connection.cc:537] server connection from 127.0.0.1:50210 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:03:42.979 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.256776 32580 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 24794a00822242cba0e345d7ec9fd95f)
21:03:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.256991 32580 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.257208 32581 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 24794a00822242cba0e345d7ec9fd95f)
21:03:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.257392 32581 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:43.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.258479 32581 leader_election.cc:290] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 0a522657a17a4489b27da69a1907160c (127.31.1.126:41315), 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201)
21:03:43.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.258497 32580 leader_election.cc:290] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6bf55660ec6046509a1e498a52e393e5 (127.31.1.125:33977), 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201)
21:03:43.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.258977 31801 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6bf55660ec6046509a1e498a52e393e5" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "0a522657a17a4489b27da69a1907160c" is_pre_election: true
21:03:43.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.259065 31880 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0a522657a17a4489b27da69a1907160c" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "6bf55660ec6046509a1e498a52e393e5" is_pre_election: true
21:03:43.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.259191 31801 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6bf55660ec6046509a1e498a52e393e5 in term 1.
21:03:43.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.259253 31880 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0a522657a17a4489b27da69a1907160c in term 1.
21:03:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.259713 31778 leader_election.cc:304] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0a522657a17a4489b27da69a1907160c, 6bf55660ec6046509a1e498a52e393e5; no voters:
21:03:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.260239 32580 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 1 FOLLOWER]: Leader pre-election won for term 2
21:03:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.260242 31857 leader_election.cc:304] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0a522657a17a4489b27da69a1907160c, 6bf55660ec6046509a1e498a52e393e5; no voters:
21:03:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.260445 32580 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 1 FOLLOWER]: Starting leader election (detected failure of leader 24794a00822242cba0e345d7ec9fd95f)
21:03:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.260627 32581 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:03:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.260643 32580 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 1 FOLLOWER]: Advancing to term 2
21:03:43.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.260890 32581 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 24794a00822242cba0e345d7ec9fd95f)
21:03:43.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.261108 32581 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 1 FOLLOWER]: Advancing to term 2
21:03:43.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.263409 32580 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.263998 32581 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:43.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.264564 31880 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0a522657a17a4489b27da69a1907160c" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "6bf55660ec6046509a1e498a52e393e5"
21:03:43.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.264818 31880 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 0a522657a17a4489b27da69a1907160c in current term 2: Already voted for candidate 6bf55660ec6046509a1e498a52e393e5 in this term.
21:03:43.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.264915 32581 leader_election.cc:290] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 2 election: Requested vote from peers 0a522657a17a4489b27da69a1907160c (127.31.1.126:41315), 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201)
21:03:43.265 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.265482 32580 leader_election.cc:290] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 2 election: Requested vote from peers 6bf55660ec6046509a1e498a52e393e5 (127.31.1.125:33977), 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201)
21:03:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.266494 31801 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6bf55660ec6046509a1e498a52e393e5" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "0a522657a17a4489b27da69a1907160c"
21:03:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.266635 31857 leader_election.cc:336] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.266712 31801 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 6bf55660ec6046509a1e498a52e393e5 in current term 2: Already voted for candidate 0a522657a17a4489b27da69a1907160c in this term.
21:03:43.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.267892 31857 leader_election.cc:336] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.268 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.268 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.268383 31857 leader_election.cc:304] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6bf55660ec6046509a1e498a52e393e5; no voters: 0a522657a17a4489b27da69a1907160c, 24794a00822242cba0e345d7ec9fd95f
21:03:43.268 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.31.1.124:41201(127.31.1.124:41201) for tablet Kudu Master: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.269119 32581 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:03:43.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.270068 31776 leader_election.cc:336] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.270475 31776 leader_election.cc:336] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.270588 31776 leader_election.cc:304] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0a522657a17a4489b27da69a1907160c; no voters: 24794a00822242cba0e345d7ec9fd95f, 6bf55660ec6046509a1e498a52e393e5
21:03:43.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.270867 32580 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:03:43.280 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.281 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.282 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.282 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.282 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.300 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.320 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.321 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.321 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.322 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.340 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.341 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.341 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.341 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.360 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.361 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.361 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.361 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.362 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.380 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.381 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.381 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.381 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.481 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.482 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.482 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.482 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.482 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.483 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.31.1.124:41201, [8ms] received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [8ms] delaying RPC due to: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [21ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [22ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [41ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [101ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [101ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [222ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [222ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, deferred=Deferred@2107991608(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/48695226@1b24376 -> (continuation of Deferred@1513604910 after retry RPC after error@1448760085) -> (continuation of Deferred@1600320 after retry RPC after error@321003974) -> (continuation of Deferred@2001246779 after retry RPC after error@1930341692) -> (continuation of Deferred@806328950 after retry RPC after error@841068960) -> (continuation of Deferred@25918621 after retry RPC after error@124405469) -> (continuation of Deferred@1266331756 after retry RPC after error@1953084434), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/518121423@2edc4196 -> (continuation of Deferred@1513604910 after retry RPC after error@1448760085) -> (continuation of Deferred@1600320 after retry RPC after error@321003974) -> (continuation of Deferred@2001246779 after retry RPC after error@1930341692) -> (continuation of Deferred@806328950 after retry RPC after error@841068960) -> (continuation of Deferred@25918621 after retry RPC after error@124405469) -> (continuation of Deferred@1266331756 after retry RPC after error@1953084434)))
21:03:43.484 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.31.1.124:41201, [8ms] received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [8ms] delaying RPC due to: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [19ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [21ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [22ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [41ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [101ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [101ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [222ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [222ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, deferred=Deferred@2107991608(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/48695226@1b24376 -> (continuation of Deferred@1513604910 after retry RPC after error@1448760085) -> (continuation of Deferred@1600320 after retry RPC after error@321003974) -> (continuation of Deferred@2001246779 after retry RPC after error@1930341692) -> (continuation of Deferred@806328950 after retry RPC after error@841068960) -> (continuation of Deferred@25918621 after retry RPC after error@124405469) -> (continuation of Deferred@1266331756 after retry RPC after error@1953084434), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/518121423@2edc4196 -> (continuation of Deferred@1513604910 after retry RPC after error@1448760085) -> (continuation of Deferred@1600320 after retry RPC after error@321003974) -> (continuation of Deferred@2001246779 after retry RPC after error@1930341692) -> (continuation of Deferred@806328950 after retry RPC after error@841068960) -> (continuation of Deferred@25918621 after retry RPC after error@124405469) -> (continuation of Deferred@1266331756 after retry RPC after error@1953084434)))
21:03:43.501 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.502 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.502 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.502 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.521 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.522 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.522 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.522 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.522 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.541 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.542 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.542 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.542 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.543 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.560 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.561 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.561 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.561 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.562 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.580 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.581 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.581 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.581 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.620 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.621 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.621 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.621 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.632056 32581 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:43.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.632303 32581 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:43.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.633008 32581 leader_election.cc:290] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 0a522657a17a4489b27da69a1907160c (127.31.1.126:41315), 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201)
21:03:43.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.633700 31801 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6bf55660ec6046509a1e498a52e393e5" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "0a522657a17a4489b27da69a1907160c" is_pre_election: true
21:03:43.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.633894 32580 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.634047 32580 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.634336 31801 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6bf55660ec6046509a1e498a52e393e5 in term 2.
21:03:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.634654 32580 leader_election.cc:290] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 6bf55660ec6046509a1e498a52e393e5 (127.31.1.125:33977), 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201)
21:03:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.634905 31857 leader_election.cc:336] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.635191 31857 leader_election.cc:304] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0a522657a17a4489b27da69a1907160c, 6bf55660ec6046509a1e498a52e393e5; no voters: 24794a00822242cba0e345d7ec9fd95f
21:03:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.635288 31776 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111) [suppressed 9 similar messages]
21:03:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.635301 31880 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0a522657a17a4489b27da69a1907160c" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "6bf55660ec6046509a1e498a52e393e5" is_pre_election: true
21:03:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.635495 31880 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0a522657a17a4489b27da69a1907160c in term 2.
21:03:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.635668 32581 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 2 FOLLOWER]: Leader pre-election won for term 3
21:03:43.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.635921 32581 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:43.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.635967 31778 leader_election.cc:304] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0a522657a17a4489b27da69a1907160c, 6bf55660ec6046509a1e498a52e393e5; no voters:
21:03:43.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.636055 32581 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 2 FOLLOWER]: Advancing to term 3
21:03:43.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.636268 32580 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 2 FOLLOWER]: Leader pre-election won for term 3
21:03:43.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.636274 31776 leader_election.cc:336] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.636380 32580 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:43.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.636509 32580 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 2 FOLLOWER]: Advancing to term 3
21:03:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.638200 32581 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.638234 32580 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:43.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.638728 32580 leader_election.cc:290] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 3 election: Requested vote from peers 6bf55660ec6046509a1e498a52e393e5 (127.31.1.125:33977), 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201)
21:03:43.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.638818 32581 leader_election.cc:290] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 3 election: Requested vote from peers 0a522657a17a4489b27da69a1907160c (127.31.1.126:41315), 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201)
21:03:43.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.639181 31880 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0a522657a17a4489b27da69a1907160c" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "6bf55660ec6046509a1e498a52e393e5"
21:03:43.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.639400 31880 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 3 FOLLOWER]: Leader election vote request: Denying vote to candidate 0a522657a17a4489b27da69a1907160c in current term 3: Already voted for candidate 6bf55660ec6046509a1e498a52e393e5 in this term.
21:03:43.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.639369 31801 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6bf55660ec6046509a1e498a52e393e5" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "0a522657a17a4489b27da69a1907160c"
21:03:43.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.639592 31801 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 3 FOLLOWER]: Leader election vote request: Denying vote to candidate 6bf55660ec6046509a1e498a52e393e5 in current term 3: Already voted for candidate 0a522657a17a4489b27da69a1907160c in this term.
21:03:43.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.640386 31857 leader_election.cc:336] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.640538 31776 leader_election.cc:336] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.640518 31857 leader_election.cc:304] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [CANDIDATE]: Term 3 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6bf55660ec6046509a1e498a52e393e5; no voters: 0a522657a17a4489b27da69a1907160c, 24794a00822242cba0e345d7ec9fd95f
21:03:43.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.640663 31776 leader_election.cc:304] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 3 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0a522657a17a4489b27da69a1907160c; no voters: 24794a00822242cba0e345d7ec9fd95f, 6bf55660ec6046509a1e498a52e393e5
21:03:43.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.640794 32581 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 3 FOLLOWER]: Leader election lost for term 3. Reason: could not achieve majority
21:03:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.640933 32580 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 3 FOLLOWER]: Leader election lost for term 3. Reason: could not achieve majority
21:03:43.680 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.682 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.682 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.682 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.682 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.683 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [22ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [41ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [43ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [122ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [181ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [183ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, deferred=Deferred@941530228(state=PENDING, result=null, callback=(continuation of Deferred@1645382528 after retry RPC after error@1890364889) -> (continuation of Deferred@725663245 after retry RPC after error@1735485997) -> (continuation of Deferred@290038728 after retry RPC after error@77465181) -> (continuation of Deferred@713056277 after retry RPC after error@1179408234) -> (continuation of Deferred@440920968 after retry RPC after error@1847569011) -> (continuation of Deferred@1194786669 after retry RPC after error@1675954150), errback=(continuation of Deferred@1645382528 after retry RPC after error@1890364889) -> (continuation of Deferred@725663245 after retry RPC after error@1735485997) -> (continuation of Deferred@290038728 after retry RPC after error@77465181) -> (continuation of Deferred@713056277 after retry RPC after error@1179408234) -> (continuation of Deferred@440920968 after retry RPC after error@1847569011) -> (continuation of Deferred@1194786669 after retry RPC after error@1675954150)))
21:03:43.683 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [22ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [41ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [43ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [122ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [181ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [183ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, deferred=Deferred@941530228(state=PENDING, result=null, callback=(continuation of Deferred@1645382528 after retry RPC after error@1890364889) -> (continuation of Deferred@725663245 after retry RPC after error@1735485997) -> (continuation of Deferred@290038728 after retry RPC after error@77465181) -> (continuation of Deferred@713056277 after retry RPC after error@1179408234) -> (continuation of Deferred@440920968 after retry RPC after error@1847569011) -> (continuation of Deferred@1194786669 after retry RPC after error@1675954150), errback=(continuation of Deferred@1645382528 after retry RPC after error@1890364889) -> (continuation of Deferred@725663245 after retry RPC after error@1735485997) -> (continuation of Deferred@290038728 after retry RPC after error@77465181) -> (continuation of Deferred@713056277 after retry RPC after error@1179408234) -> (continuation of Deferred@440920968 after retry RPC after error@1847569011) -> (continuation of Deferred@1194786669 after retry RPC after error@1675954150)))
21:03:43.700 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.702 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.702 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.702 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.702 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.720 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.721 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.721 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.721 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.740 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.742 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.742 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.742 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.743 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.760 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.762 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.762 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.762 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.762 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.772249 32482 heartbeater.cc:646] Failed to heartbeat to 127.31.1.124:41201 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.777976 32341 heartbeater.cc:646] Failed to heartbeat to 127.31.1.124:41201 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.780 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.781 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.781 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.781 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.782 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.791199 32201 heartbeater.cc:646] Failed to heartbeat to 127.31.1.124:41201 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.800 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.801 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.801 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.801 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.820 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.821 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.821 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.821 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.822 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.900 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.901 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.901 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.902 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.903 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [1ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [2ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [21ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [43ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [62ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [101ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [101ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [201ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [202ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, deferred=Deferred@532506002(state=PENDING, result=null, callback=(continuation of Deferred@1075553567 after retry RPC after error@705175888) -> (continuation of Deferred@1173635901 after retry RPC after error@831602660) -> (continuation of Deferred@742348692 after retry RPC after error@1205264866) -> (continuation of Deferred@542945166 after retry RPC after error@411753086) -> (continuation of Deferred@1314897164 after retry RPC after error@1047956692) -> (continuation of Deferred@2073261131 after retry RPC after error@889460845) -> (continuation of Deferred@109081628 after retry RPC after error@203049012), errback=(continuation of Deferred@1075553567 after retry RPC after error@705175888) -> (continuation of Deferred@1173635901 after retry RPC after error@831602660) -> (continuation of Deferred@742348692 after retry RPC after error@1205264866) -> (continuation of Deferred@542945166 after retry RPC after error@411753086) -> (continuation of Deferred@1314897164 after retry RPC after error@1047956692) -> (continuation of Deferred@2073261131 after retry RPC after error@889460845) -> (continuation of Deferred@109081628 after retry RPC after error@203049012)))
21:03:43.904 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [1ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [2ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [21ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [43ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [61ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [62ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [101ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [101ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [201ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [202ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [202ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, deferred=Deferred@532506002(state=PENDING, result=null, callback=(continuation of Deferred@1075553567 after retry RPC after error@705175888) -> (continuation of Deferred@1173635901 after retry RPC after error@831602660) -> (continuation of Deferred@742348692 after retry RPC after error@1205264866) -> (continuation of Deferred@542945166 after retry RPC after error@411753086) -> (continuation of Deferred@1314897164 after retry RPC after error@1047956692) -> (continuation of Deferred@2073261131 after retry RPC after error@889460845) -> (continuation of Deferred@109081628 after retry RPC after error@203049012), errback=(continuation of Deferred@1075553567 after retry RPC after error@705175888) -> (continuation of Deferred@1173635901 after retry RPC after error@831602660) -> (continuation of Deferred@742348692 after retry RPC after error@1205264866) -> (continuation of Deferred@542945166 after retry RPC after error@411753086) -> (continuation of Deferred@1314897164 after retry RPC after error@1047956692) -> (continuation of Deferred@2073261131 after retry RPC after error@889460845) -> (continuation of Deferred@109081628 after retry RPC after error@203049012)))
21:03:43.920 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.921 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.921 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.922 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.922 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.938027 32083 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.183s from last keepalive heartbeat (effective timeout is 1.000s)
21:03:43.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.939281 32109 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.31.1.124:41201)
21:03:43.940 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.941 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.941 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.941 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.942 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.960 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.961 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.961 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.961 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.962 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.979482 32580 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:43.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.979678 32580 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:43.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.980440 32580 leader_election.cc:290] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 6bf55660ec6046509a1e498a52e393e5 (127.31.1.125:33977), 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201)
21:03:43.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.980775 31880 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0a522657a17a4489b27da69a1907160c" candidate_term: 4 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "6bf55660ec6046509a1e498a52e393e5" is_pre_election: true
21:03:43.981 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.981003 31880 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0a522657a17a4489b27da69a1907160c in term 3.
21:03:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.981400 31778 leader_election.cc:304] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0a522657a17a4489b27da69a1907160c, 6bf55660ec6046509a1e498a52e393e5; no voters:
21:03:43.982 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.982 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.982 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.982060 32580 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 3 FOLLOWER]: Leader pre-election won for term 4
21:03:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.982185 32580 raft_consensus.cc:493] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.982287 32580 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 3 FOLLOWER]: Advancing to term 4
21:03:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.982313 31776 leader_election.cc:336] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.984293 32580 raft_consensus.cc:515] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:43.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.984834 32580 leader_election.cc:290] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 4 election: Requested vote from peers 6bf55660ec6046509a1e498a52e393e5 (127.31.1.125:33977), 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201)
21:03:43.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.985281 31880 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0a522657a17a4489b27da69a1907160c" candidate_term: 4 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "6bf55660ec6046509a1e498a52e393e5"
21:03:43.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.985481 31880 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 3 FOLLOWER]: Advancing to term 4
21:03:43.985 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:43.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:43.986198 31776 leader_election.cc:336] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111)
21:03:43.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.987562 31880 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0a522657a17a4489b27da69a1907160c in term 4.
21:03:43.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.987962 31778 leader_election.cc:304] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0a522657a17a4489b27da69a1907160c, 6bf55660ec6046509a1e498a52e393e5; no voters: 24794a00822242cba0e345d7ec9fd95f
21:03:43.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.988217 32580 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 4 FOLLOWER]: Leader election won for term 4
21:03:43.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.988613 32580 raft_consensus.cc:697] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [term 4 LEADER]: Becoming Leader. State: Replica: 0a522657a17a4489b27da69a1907160c, State: Running, Role: LEADER
21:03:43.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.989064 32580 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } }
21:03:43.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.990803 32601 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0a522657a17a4489b27da69a1907160c. Latest consensus state: current_term: 4 leader_uuid: "0a522657a17a4489b27da69a1907160c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:43.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.991114 32601 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [sys.catalog]: This master's current role is: LEADER
21:03:43.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.991742 32603 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:03:43.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.994096 32603 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback [id=4b1bd3ea937a4e26ae7fdf112e5fadaa]
21:03:43.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.994748 32603 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=fdcb6236b99b439ba664548dbfc6b601]
21:03:43.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.996397 32603 tablet_loader.cc:96] loaded metadata for tablet aa55302b73564eb887f0a64a87542411 (table txn_manager_fallback [id=4b1bd3ea937a4e26ae7fdf112e5fadaa])
21:03:43.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.996789 32603 tablet_loader.cc:96] loaded metadata for tablet bff3b2b38a50457497ad56d0ced46e75 (table kudu_system.kudu_transactions [id=fdcb6236b99b439ba664548dbfc6b601])
21:03:43.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.997098 32603 tablet_loader.cc:96] loaded metadata for tablet e9c747530e3a4b36864f2ab09e4d7436 (table txn_manager_fallback [id=4b1bd3ea937a4e26ae7fdf112e5fadaa])
21:03:43.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.997305 32603 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:03:43.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.998142 32603 catalog_manager.cc:1269] Loaded cluster ID: 3ad62dfb1f6141a0bf90bca6aa25508d
21:03:43.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.998224 32603 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:03:43.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:43.999293 32603 catalog_manager.cc:1514] Loading token signing keys...
21:03:44.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.000100 32603 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c: Loaded TSK: 0
21:03:44.000 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:44.001 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:44.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.001035 32603 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:03:44.001 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:44.002 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:44.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201; will retry
21:03:44.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:44.014706 32589 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
21:03:44.040 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.31.1.124:41201(127.31.1.124:41201)] connecting to peer
21:03:44.041 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:44.042 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.124:41201(127.31.1.124:41201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:44.042 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.1.124:41201: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201
21:03:44.042 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:03:44.043 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=6, TimeoutTracker(timeout=250, elapsed=122), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [1ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [21ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [41ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [41ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [65ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [65ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [82ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.31.1.126:41315,127.31.1.125:33977,127.31.1.124:41201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.126:41315, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.125:33977, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.1.124:41201, [122ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.124:41201: Network error: Failed to connect to peer master-127.31.1.124:41201(127.31.1.124:41201): Connection refused: /127.31.1.124:41201, [122ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.125:33977: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.31.1.126:41315: OK, deferred=Deferred@1009932772(state=PENDING, result=null, callback=(continuation of Deferred@1370219423 after retry RPC after error@1466482865) -> (continuation of Deferred@244388182 after retry RPC after error@549618713) -> (continuation of Deferred@779824261 after retry RPC after error@1057426611) -> (continuation of Deferred@958050724 after retry RPC after error@231479506) -> (continuation of Deferred@2049180394 after retry RPC after error@2101472137), errback=(continuation of Deferred@1370219423 after retry RPC after error@1466482865) -> (continuation of Deferred@244388182 after retry RPC after error@549618713) -> (continuation of Deferred@779824261 after retry RPC after error@1057426611) -> (continuation of Deferred@958050724 after retry RPC after error@231479506) -> (continuation of Deferred@2049180394 after retry RPC after error@2101472137))) after lookup
21:03:44.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:44.094415 31776 consensus_peers.cc:597] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c -> Peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Couldn't send request to peer 24794a00822242cba0e345d7ec9fd95f. Status: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.098193 31790 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50234:
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:03:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:03:44.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:44.099879 31790 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:03:44.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.105530 31880 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [term 4 FOLLOWER]: Refusing update from remote peer 0a522657a17a4489b27da69a1907160c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 4 index: 12. (index mismatch)
21:03:44.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.106151 32601 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
21:03:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.108083 32581 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0a522657a17a4489b27da69a1907160c. Latest consensus state: current_term: 4 leader_uuid: "0a522657a17a4489b27da69a1907160c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.108345 32581 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [sys.catalog]: This master's current role is: FOLLOWER
21:03:44.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.110702 32581 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "0a522657a17a4489b27da69a1907160c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:44.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.110873 32581 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6bf55660ec6046509a1e498a52e393e5 [sys.catalog]: This master's current role is: FOLLOWER
21:03:44.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.111754 32580 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "0a522657a17a4489b27da69a1907160c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0a522657a17a4489b27da69a1907160c" member_type: VOTER last_known_addr { host: "127.31.1.126" port: 41315 } } peers { permanent_uuid: "6bf55660ec6046509a1e498a52e393e5" member_type: VOTER last_known_addr { host: "127.31.1.125" port: 33977 } } peers { permanent_uuid: "24794a00822242cba0e345d7ec9fd95f" member_type: VOTER last_known_addr { host: "127.31.1.124" port: 41201 } } }
21:03:44.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.112021 32580 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c [sys.catalog]: This master's current role is: LEADER
21:03:44.119 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 34cad66201db44ecb5a55678bd60a0e6 (127.31.1.65:46755): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1765746222 last_transition_timestamp: 1765746224
21:03:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.411036 31760 master_runner.cc:186] Error getting master registration for 127.31.1.124:41201: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111),
21:03:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.411234 31760 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:03:44.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:44.624689 31776 consensus_peers.cc:597] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c -> Peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Couldn't send request to peer 24794a00822242cba0e345d7ec9fd95f. Status: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:03:44.990 [DEBUG - main] (Connection.java:575) [peer master-127.31.1.124:41201(127.31.1.124:41201)] disconnecting while in state TERMINATED
21:03:44.990 [DEBUG - main] (Connection.java:575) [peer master-127.31.1.125:33977(127.31.1.125:33977)] disconnecting while in state READY
21:03:44.990 [DEBUG - main] (Connection.java:575) [peer 34cad66201db44ecb5a55678bd60a0e6(127.31.1.65:46755)] disconnecting while in state READY
21:03:44.990 [DEBUG - main] (Connection.java:575) [peer master-127.31.1.126:41315(127.31.1.126:41315)] disconnecting while in state READY
21:03:44.991 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.31.1.126:41315(127.31.1.126:41315)] handling channelInactive
21:03:44.991 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.1.126:41315(127.31.1.126:41315)] cleaning up while in state READY due to: connection closed
21:03:44.991 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:03:44.992 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.31.1.125:33977(127.31.1.125:33977)] handling channelInactive
21:03:44.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.1.125:33977(127.31.1.125:33977)] cleaning up while in state READY due to: connection closed
21:03:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:44.992416 31749 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task6gQn2r/build/asan/bin/kudu pid 32044
21:03:44.992 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 34cad66201db44ecb5a55678bd60a0e6(127.31.1.65:46755)] handling channelInactive
21:03:44.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 34cad66201db44ecb5a55678bd60a0e6(127.31.1.65:46755)] cleaning up while in state READY due to: connection closed
21:03:45.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.001061 32196 generic_service.cc:196] Checking for leaks (request via RPC)
21:03:45.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:45.120635 31776 consensus_peers.cc:597] T 00000000000000000000000000000000 P 0a522657a17a4489b27da69a1907160c -> Peer 24794a00822242cba0e345d7ec9fd95f (127.31.1.124:41201): Couldn't send request to peer 24794a00822242cba0e345d7ec9fd95f. Status: Network error: Client connection negotiation failed: client connection to 127.31.1.124:41201: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
21:03:45.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.127235 31749 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task6gQn2r/build/asan/bin/kudu with pid 32044
21:03:45.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.149183 31749 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task6gQn2r/build/asan/bin/kudu pid 32204
21:03:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.155700 32336 generic_service.cc:196] Checking for leaks (request via RPC)
21:03:45.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.253932 31749 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task6gQn2r/build/asan/bin/kudu with pid 32204
21:03:45.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:45.270856 32361 connection.cc:537] server connection from 127.31.1.66:43021 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:03:45.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.271137 31749 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task6gQn2r/build/asan/bin/kudu pid 32346
21:03:45.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.277860 32477 generic_service.cc:196] Checking for leaks (request via RPC)
21:03:45.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.376294 31749 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task6gQn2r/build/asan/bin/kudu with pid 32346
21:03:45.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.396152 31749 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task6gQn2r/build/asan/bin/kudu pid 31760
21:03:45.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.397269 31831 generic_service.cc:196] Checking for leaks (request via RPC)
21:03:45.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.513664 31749 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task6gQn2r/build/asan/bin/kudu with pid 31760
21:03:45.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:03:45.534538 31856 connection.cc:537] server connection from 127.0.0.1:34364 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:03:45.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.534878 31749 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-task6gQn2r/build/asan/bin/kudu pid 31838
21:03:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.535923 31912 generic_service.cc:196] Checking for leaks (request via RPC)
21:03:45.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:03:45.645171 31749 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task6gQn2r/build/asan/bin/kudu with pid 31838
21:03:45.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-14T21:03:45Z chronyd exiting
21:03:45.805 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 34cad66201db44ecb5a55678bd60a0e6 (127.31.1.65:46755): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1765746222 last_transition_timestamp: 1765746224
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 34cad66201db44ecb5a55678bd60a0e6 (127.31.1.65:46755): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1765746222 last_transition_timestamp: 1765746224
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more